ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter3/contextual-retrieval/evaluation/dataset_builder.py

@@ -0,0 +1,317 @@
+"""Build evaluation dataset from Chinese legal documents"""
+
+import json
+import logging
+from typing import List, Dict, Any
+from pathlib import Path
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class LegalDatasetBuilder:
+    """Build evaluation dataset for Chinese legal Q&A"""
+    
+    def __init__(self):
+        self.simple_cases = []
+        self.complex_cases = []
+        
+    def create_simple_cases(self) -> List[Dict[str, Any]]:
+        """Create simple direct legal questions"""
+        simple_cases = [
+            {
+                "id": "simple_1",
+                "question": "故意杀人罪判几年？",
+                "expected_keywords": ["死刑", "无期徒刑", "十年以上有期徒刑"],
+                "reference": "《中华人民共和国刑法》第二百三十二条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_2",
+                "question": "盗窃罪的立案标准是什么？",
+                "expected_keywords": ["一千元", "三千元", "数额较大"],
+                "reference": "《中华人民共和国刑法》第二百六十四条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_3", 
+                "question": "醉酒驾驶机动车如何处罚？",
+                "expected_keywords": ["拘役", "罚金", "吊销驾照"],
+                "reference": "《中华人民共和国刑法》第一百三十三条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_4",
+                "question": "诈骗罪的量刑标准是什么？",
+                "expected_keywords": ["三年以下", "三年以上十年以下", "十年以上"],
+                "reference": "《中华人民共和国刑法》第二百六十六条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_5",
+                "question": "故意伤害罪致人重伤的处罚是什么？",
+                "expected_keywords": ["三年以上十年以下", "有期徒刑"],
+                "reference": "《中华人民共和国刑法》第二百三十四条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_6",
+                "question": "抢劫罪的加重情节有哪些？",
+                "expected_keywords": ["入户抢劫", "多次抢劫", "抢劫数额巨大"],
+                "reference": "《中华人民共和国刑法》第二百六十三条",
+                "difficulty": "medium"
+            },
+            {
+                "id": "simple_7",
+                "question": "非法拘禁罪的构成要件是什么？",
+                "expected_keywords": ["非法", "拘禁", "限制人身自由"],
+                "reference": "《中华人民共和国刑法》第二百三十八条",
+                "difficulty": "medium"
+            },
+            {
+                "id": "simple_8",
+                "question": "贪污罪的数额标准如何认定？",
+                "expected_keywords": ["三万元", "二十万元", "三百万元"],
+                "reference": "《中华人民共和国刑法》第三百八十三条",
+                "difficulty": "medium"
+            },
+            {
+                "id": "simple_9",
+                "question": "交通肇事罪的立案标准是什么？",
+                "expected_keywords": ["死亡一人", "重伤三人", "财产损失"],
+                "reference": "《中华人民共和国刑法》第一百三十三条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_10",
+                "question": "寻衅滋事罪如何处罚？",
+                "expected_keywords": ["五年以下", "有期徒刑", "拘役", "管制"],
+                "reference": "《中华人民共和国刑法》第二百九十三条",
+                "difficulty": "easy"
+            }
+        ]
+        
+        return simple_cases
+    
+    def create_complex_cases(self) -> List[Dict[str, Any]]:
+        """Create complex legal scenario questions"""
+        complex_cases = [
+            {
+                "id": "complex_1",
+                "question": """张某因与李某发生经济纠纷，持刀闯入李某家中，意图讨债。在争执过程中，张某用刀刺伤李某，
+                            导致李某重伤。同时，张某还顺手拿走了李某家中的现金5万元。请问张某的行为应如何定性？
+                            可能面临什么样的刑事处罚？""",
+                "expected_analysis": ["入户抢劫", "故意伤害", "数罪并罚"],
+                "reference": "《刑法》第二百三十四条、第二百六十三条",
+                "difficulty": "hard",
+                "requires_multi_query": True
+            },
+            {
+                "id": "complex_2",
+                "question": """王某系某国有企业财务主管，利用职务之便，通过虚开发票等手段，
+                            将公司资金200万元转入其控制的账户。后王某用该资金进行股票投资，
+                            获利50万元。案发后，王某主动退还全部赃款。请分析王某的法律责任。""",
+                "expected_analysis": ["贪污罪", "挪用公款罪", "自首情节", "退赃"],
+                "reference": "《刑法》第三百八十二条、第三百八十四条",
+                "difficulty": "hard",
+                "requires_multi_query": True
+            },
+            {
+                "id": "complex_3",
+                "question": """赵某酒后驾车，在市区超速行驶，撞倒正在过马路的行人陈某，
+                            导致陈某当场死亡。赵某见状，驾车逃离现场。第二天，在家人劝说下，
+                            赵某到公安机关投案自首。请问赵某涉嫌哪些犯罪？量刑时应考虑哪些因素？""",
+                "expected_analysis": ["交通肇事罪", "危险驾驶罪", "逃逸", "自首"],
+                "reference": "《刑法》第一百三十三条",
+                "difficulty": "hard",
+                "requires_multi_query": True
+            },
+            {
+                "id": "complex_4",
+                "question": """刘某通过网络平台发布虚假投资信息，声称可以保证高额回报，
+                            先后骗取30名投资者共计500万元。其中，刘某将200万元用于个人挥霍，
+                            300万元用于归还之前的债务。请问刘某的行为如何定性？可能的量刑是什么？""",
+                "expected_analysis": ["诈骗罪", "数额特别巨大", "多人受害"],
+                "reference": "《刑法》第二百六十六条",
+                "difficulty": "hard",
+                "requires_multi_query": True
+            },
+            {
+                "id": "complex_5",
+                "question": """孙某与钱某共谋盗窃某商场。孙某负责望风，钱某进入商场实施盗窃。
+                            钱某在盗窃过程中被保安发现，为逃跑将保安打成轻伤。
+                            最终二人盗窃财物价值8万元。请分析孙某和钱某各自的刑事责任。""",
+                "expected_analysis": ["共同犯罪", "盗窃罪", "抢劫罪", "转化犯"],
+                "reference": "《刑法》第二百六十四条、第二百六十九条",
+                "difficulty": "hard",
+                "requires_multi_query": True
+            }
+        ]
+        
+        return complex_cases
+    
+    def build_dataset(self, output_path: str = "legal_qa_dataset.json"):
+        """Build and save the complete dataset"""
+        dataset = {
+            "simple_cases": self.create_simple_cases(),
+            "complex_cases": self.create_complex_cases(),
+            "metadata": {
+                "total_cases": 15,
+                "simple_count": 10,
+                "complex_count": 5,
+                "domain": "Chinese Criminal Law",
+                "purpose": "Evaluate agentic vs non-agentic RAG performance"
+            }
+        }
+        
+        # Save dataset
+        with open(output_path, 'w', encoding='utf-8') as f:
+            json.dump(dataset, f, ensure_ascii=False, indent=2)
+        
+        logger.info(f"Dataset saved to {output_path}")
+        return dataset
+
+
+def create_legal_documents() -> List[Dict[str, str]]:
+    """Create sample legal documents for the knowledge base"""
+    documents = [
+        {
+            "doc_id": "criminal_law_homicide",
+            "title": "刑法-故意杀人罪",
+            "content": """第二百三十二条 【故意杀人罪】故意杀人的，处死刑、无期徒刑或者十年以上有期徒刑；
+            情节较轻的，处三年以上十年以下有期徒刑。
+            
+            故意杀人罪是指故意非法剥夺他人生命的行为。该罪侵犯的客体是他人的生命权。
+            法律依据是《中华人民共和国刑法》第二百三十二条。
+            
+            量刑标准：
+            1. 情节严重的：死刑、无期徒刑或十年以上有期徒刑
+            2. 情节较轻的：三年以上十年以下有期徒刑
+            
+            情节较轻通常包括：防卫过当、义愤杀人、被害人有过错等情形。"""
+        },
+        {
+            "doc_id": "criminal_law_theft",
+            "title": "刑法-盗窃罪",
+            "content": """第二百六十四条 【盗窃罪】盗窃公私财物，数额较大的，或者多次盗窃、入户盗窃、
+            携带凶器盗窃、扒窃的，处三年以下有期徒刑、拘役或者管制，并处或者单处罚金；
+            数额巨大或者有其他严重情节的，处三年以上十年以下有期徒刑，并处罚金；
+            数额特别巨大或者有其他特别严重情节的，处十年以上有期徒刑或者无期徒刑，并处罚金或者没收财产。
+            
+            盗窃罪的立案标准：
+            1. 数额较大：一般为1000元至3000元以上
+            2. 数额巨大：一般为3万元至10万元以上  
+            3. 数额特别巨大：一般为30万元至50万元以上
+            
+            特殊情形：多次盗窃（2年内3次以上）、入户盗窃、携带凶器盗窃、扒窃的，
+            不论数额大小，均构成盗窃罪。"""
+        },
+        {
+            "doc_id": "criminal_law_fraud",
+            "title": "刑法-诈骗罪",
+            "content": """第二百六十六条 【诈骗罪】诈骗公私财物，数额较大的，处三年以下有期徒刑、
+            拘役或者管制，并处或者单处罚金；数额巨大或者有其他严重情节的，
+            处三年以上十年以下有期徒刑，并处罚金；数额特别巨大或者有其他特别严重情节的，
+            处十年以上有期徒刑或者无期徒刑，并处罚金或者没收财产。
+            
+            诈骗罪的量刑标准：
+            1. 数额较大（3千元至1万元以上）：三年以下有期徒刑、拘役或者管制
+            2. 数额巨大（3万元至10万元以上）：三年以上十年以下有期徒刑
+            3. 数额特别巨大（50万元以上）：十年以上有期徒刑或者无期徒刑
+            
+            诈骗罪是指以非法占有为目的，用虚构事实或者隐瞒真相的方法，
+            骗取数额较大的公私财物的行为。"""
+        },
+        {
+            "doc_id": "criminal_law_robbery",
+            "title": "刑法-抢劫罪",
+            "content": """第二百六十三条 【抢劫罪】以暴力、胁迫或者其他方法抢劫公私财物的，
+            处三年以上十年以下有期徒刑，并处罚金；有下列情形之一的，
+            处十年以上有期徒刑、无期徒刑或者死刑，并处罚金或者没收财产：
+            
+            （一）入户抢劫的；
+            （二）在公共交通工具上抢劫的；
+            （三）抢劫银行或者其他金融机构的；
+            （四）多次抢劫或者抢劫数额巨大的；
+            （五）抢劫致人重伤、死亡的；
+            （六）冒充军警人员抢劫的；
+            （七）持枪抢劫的；
+            （八）抢劫军用物资或者抢险、救灾、救济物资的。
+            
+            抢劫罪的加重处罚情节包括上述八种情形，有其中之一的，
+            最低刑期为十年有期徒刑。"""
+        },
+        {
+            "doc_id": "criminal_law_injury",
+            "title": "刑法-故意伤害罪",
+            "content": """第二百三十四条 【故意伤害罪】故意伤害他人身体的，处三年以下有期徒刑、
+            拘役或者管制。犯前款罪，致人重伤的，处三年以上十年以下有期徒刑；
+            致人死亡或者以特别残忍手段致人重伤造成严重残疾的，处十年以上有期徒刑、
+            无期徒刑或者死刑。
+            
+            故意伤害罪的量刑：
+            1. 故意伤害致人轻伤的：三年以下有期徒刑、拘役或者管制
+            2. 故意伤害致人重伤的：三年以上十年以下有期徒刑
+            3. 故意伤害致人死亡或特别残忍手段致残的：十年以上有期徒刑、无期徒刑或死刑
+            
+            重伤标准：使人肢体残废或者毁人容貌；使人丧失听觉、视觉或者其他器官功能；
+            其他对于人身健康有重大伤害的。"""
+        },
+        {
+            "doc_id": "criminal_law_traffic",
+            "title": "刑法-交通肇事罪与危险驾驶罪",
+            "content": """第一百三十三条 【交通肇事罪】违反交通运输管理法规，因而发生重大事故，
+            致人重伤、死亡或者使公私财产遭受重大损失的，处三年以下有期徒刑或者拘役；
+            交通运输肇事后逃逸或者有其他特别恶劣情节的，处三年以上七年以下有期徒刑；
+            因逃逸致人死亡的，处七年以上有期徒刑。
+            
+            第一百三十三条之一 【危险驾驶罪】在道路上驾驶机动车，有下列情形之一的，
+            处拘役，并处罚金：
+            （一）追逐竞驶，情节恶劣的；
+            （二）醉酒驾驶机动车的；
+            （三）从事校车业务或者旅客运输，严重超过额定乘员载客，
+                  或者严重超过规定时速行驶的；
+            （四）违反危险化学品安全管理规定运输危险化学品，危及公共安全的。
+            
+            醉酒驾驶的认定标准：血液酒精含量达到80毫克/100毫升以上。"""
+        },
+        {
+            "doc_id": "criminal_law_corruption",
+            "title": "刑法-贪污罪",
+            "content": """第三百八十二条 【贪污罪】国家工作人员利用职务上的便利，侵吞、窃取、
+            骗取或者以其他手段非法占有公共财物的，是贪污罪。
+            
+            第三百八十三条 【贪污罪的处罚规定】对犯贪污罪的，根据情节轻重，分别依照下列规定处罚：
+            
+            （一）贪污数额较大或者有其他较重情节的，处三年以下有期徒刑或者拘役，并处罚金。
+            （二）贪污数额巨大或者有其他严重情节的，处三年以上十年以下有期徒刑，并处罚金或者没收财产。
+            （三）贪污数额特别巨大或者有其他特别严重情节的，处十年以上有期徒刑或者无期徒刑，
+                  并处罚金或者没收财产；数额特别巨大，并使国家和人民利益遭受特别重大损失的，
+                  处无期徒刑或者死刑，并处没收财产。
+            
+            贪污数额标准：
+            1. 数额较大：三万元以上不满二十万元
+            2. 数额巨大：二十万元以上不满三百万元
+            3. 数额特别巨大：三百万元以上"""
+        }
+    ]
+    
+    return documents
+
+
+if __name__ == "__main__":
+    # Build evaluation dataset
+    builder = LegalDatasetBuilder()
+    dataset = builder.build_dataset("legal_qa_dataset.json")
+    
+    print(f"Dataset created with {len(dataset['simple_cases'])} simple cases and {len(dataset['complex_cases'])} complex cases")
+    
+    # Create legal documents
+    documents = create_legal_documents()
+    
+    # Save documents
+    with open("legal_documents.json", 'w', encoding='utf-8') as f:
+        json.dump(documents, f, ensure_ascii=False, indent=2)
+    
+    print(f"Created {len(documents)} legal documents for knowledge base")

chapter3/contextual-retrieval/evaluation/evaluate.py

@@ -0,0 +1,396 @@
+"""Evaluation framework for Agentic RAG system"""
+
+import json
+import logging
+import time
+from typing import List, Dict, Any, Optional
+from pathlib import Path
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from config import Config
+from agent import AgenticRAG
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class RAGEvaluator:
+    """Evaluate RAG system performance"""
+    
+    def __init__(self, config: Optional[Config] = None):
+        self.config = config or Config.from_env()
+        self.agent = AgenticRAG(self.config)
+        self.results = {
+            "agentic": [],
+            "non_agentic": []
+        }
+        
+    def load_dataset(self, dataset_path: str) -> Dict[str, Any]:
+        """Load evaluation dataset"""
+        with open(dataset_path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    
+    def evaluate_response(self, 
+                         response: str, 
+                         test_case: Dict[str, Any]) -> Dict[str, Any]:
+        """Evaluate a single response"""
+        evaluation = {
+            "case_id": test_case["id"],
+            "question": test_case["question"],
+            "response": response,
+            "metrics": {}
+        }
+        
+        # Check for expected keywords (for simple cases)
+        if "expected_keywords" in test_case:
+            keywords_found = []
+            keywords_missing = []
+            
+            for keyword in test_case["expected_keywords"]:
+                if keyword.lower() in response.lower():
+                    keywords_found.append(keyword)
+                else:
+                    keywords_missing.append(keyword)
+            
+            evaluation["metrics"]["keyword_recall"] = len(keywords_found) / len(test_case["expected_keywords"])
+            evaluation["metrics"]["keywords_found"] = keywords_found
+            evaluation["metrics"]["keywords_missing"] = keywords_missing
+        
+        # Check for analysis points (for complex cases)
+        if "expected_analysis" in test_case:
+            analysis_found = []
+            analysis_missing = []
+            
+            for point in test_case["expected_analysis"]:
+                if point.lower() in response.lower():
+                    analysis_found.append(point)
+                else:
+                    analysis_missing.append(point)
+            
+            evaluation["metrics"]["analysis_recall"] = len(analysis_found) / len(test_case["expected_analysis"])
+            evaluation["metrics"]["analysis_found"] = analysis_found
+            evaluation["metrics"]["analysis_missing"] = analysis_missing
+        
+        # Check for citations
+        citation_count = response.count("[Doc:") + response.count("[Chunk:")
+        evaluation["metrics"]["has_citations"] = citation_count > 0
+        evaluation["metrics"]["citation_count"] = citation_count
+        
+        # Response length
+        evaluation["metrics"]["response_length"] = len(response)
+        
+        # Check if response indicates no answer
+        no_answer_indicators = ["无法回答", "没有找到", "知识库中没有", "cannot answer", "not found"]
+        evaluation["metrics"]["gave_answer"] = not any(indicator in response.lower() for indicator in no_answer_indicators)
+        
+        return evaluation
+    
+    def run_test_case(self, test_case: Dict[str, Any], mode: str = "agentic") -> Dict[str, Any]:
+        """Run a single test case"""
+        logger.info(f"Running {mode} mode for case {test_case['id']}")
+        
+        start_time = time.time()
+        
+        try:
+            if mode == "agentic":
+                response = self.agent.query(test_case["question"], stream=False)
+            else:
+                response = self.agent.query_non_agentic(test_case["question"], stream=False)
+            
+            elapsed_time = time.time() - start_time
+            
+            # Clear history for next test
+            self.agent.clear_history()
+            
+            # Evaluate response
+            evaluation = self.evaluate_response(response, test_case)
+            evaluation["mode"] = mode
+            evaluation["elapsed_time"] = elapsed_time
+            evaluation["difficulty"] = test_case.get("difficulty", "unknown")
+            evaluation["success"] = True
+            
+        except Exception as e:
+            logger.error(f"Error in test case {test_case['id']}: {e}")
+            evaluation = {
+                "case_id": test_case["id"],
+                "question": test_case["question"],
+                "mode": mode,
+                "success": False,
+                "error": str(e),
+                "elapsed_time": time.time() - start_time
+            }
+        
+        return evaluation
+    
+    def run_evaluation(self, dataset_path: str, output_dir: str = "results"):
+        """Run full evaluation"""
+        # Load dataset
+        dataset = self.load_dataset(dataset_path)
+        
+        # Create output directory
+        output_path = Path(output_dir)
+        output_path.mkdir(exist_ok=True)
+        
+        # Combine all test cases
+        all_cases = dataset["simple_cases"] + dataset["complex_cases"]
+        
+        # Run agentic mode
+        logger.info("=" * 60)
+        logger.info("Running AGENTIC mode evaluation")
+        logger.info("=" * 60)
+        
+        agentic_results = []
+        for test_case in all_cases:
+            result = self.run_test_case(test_case, mode="agentic")
+            agentic_results.append(result)
+            time.sleep(1)  # Rate limiting
+        
+        # Run non-agentic mode
+        logger.info("=" * 60)
+        logger.info("Running NON-AGENTIC mode evaluation")
+        logger.info("=" * 60)
+        
+        non_agentic_results = []
+        for test_case in all_cases:
+            result = self.run_test_case(test_case, mode="non_agentic")
+            non_agentic_results.append(result)
+            time.sleep(1)  # Rate limiting
+        
+        # Compute aggregate metrics
+        agentic_metrics = self.compute_aggregate_metrics(agentic_results)
+        non_agentic_metrics = self.compute_aggregate_metrics(non_agentic_results)
+        
+        # Save results
+        results = {
+            "dataset": dataset_path,
+            "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
+            "config": {
+                "llm_provider": self.config.llm.provider,
+                "llm_model": self.agent.model,
+                "kb_type": self.config.knowledge_base.type.value
+            },
+            "agentic": {
+                "results": agentic_results,
+                "metrics": agentic_metrics
+            },
+            "non_agentic": {
+                "results": non_agentic_results,
+                "metrics": non_agentic_metrics
+            },
+            "comparison": self.compare_modes(agentic_metrics, non_agentic_metrics)
+        }
+        
+        # Save to file
+        output_file = output_path / f"evaluation_results_{time.strftime('%Y%m%d_%H%M%S')}.json"
+        with open(output_file, 'w', encoding='utf-8') as f:
+            json.dump(results, f, ensure_ascii=False, indent=2)
+        
+        logger.info(f"Results saved to {output_file}")
+        
+        # Print summary
+        self.print_summary(results)
+        
+        return results
+    
+    def compute_aggregate_metrics(self, results: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Compute aggregate metrics from results"""
+        metrics = {
+            "total_cases": len(results),
+            "successful_cases": sum(1 for r in results if r.get("success", False)),
+            "failed_cases": sum(1 for r in results if not r.get("success", False)),
+            "average_time": 0,
+            "total_time": 0
+        }
+        
+        # Separate by difficulty
+        simple_results = [r for r in results if r.get("difficulty") == "easy"]
+        medium_results = [r for r in results if r.get("difficulty") == "medium"]
+        hard_results = [r for r in results if r.get("difficulty") == "hard"]
+        
+        # Compute metrics for successful cases
+        successful_results = [r for r in results if r.get("success", False)]
+        
+        if successful_results:
+            # Time metrics
+            times = [r["elapsed_time"] for r in successful_results]
+            metrics["average_time"] = sum(times) / len(times)
+            metrics["total_time"] = sum(times)
+            metrics["min_time"] = min(times)
+            metrics["max_time"] = max(times)
+            
+            # Response quality metrics
+            metrics["cases_with_citations"] = sum(1 for r in successful_results 
+                                                 if r.get("metrics", {}).get("has_citations", False))
+            metrics["cases_gave_answer"] = sum(1 for r in successful_results 
+                                              if r.get("metrics", {}).get("gave_answer", False))
+            
+            # Average response length
+            lengths = [r.get("metrics", {}).get("response_length", 0) for r in successful_results]
+            metrics["average_response_length"] = sum(lengths) / len(lengths) if lengths else 0
+            
+            # Keyword/analysis recall (for cases that have them)
+            keyword_recalls = [r["metrics"]["keyword_recall"] for r in successful_results 
+                              if "keyword_recall" in r.get("metrics", {})]
+            if keyword_recalls:
+                metrics["average_keyword_recall"] = sum(keyword_recalls) / len(keyword_recalls)
+            
+            analysis_recalls = [r["metrics"]["analysis_recall"] for r in successful_results 
+                               if "analysis_recall" in r.get("metrics", {})]
+            if analysis_recalls:
+                metrics["average_analysis_recall"] = sum(analysis_recalls) / len(analysis_recalls)
+        
+        # Metrics by difficulty
+        for difficulty, diff_results in [("easy", simple_results), ("medium", medium_results), ("hard", hard_results)]:
+            if diff_results:
+                successful = [r for r in diff_results if r.get("success", False)]
+                metrics[f"{difficulty}_success_rate"] = len(successful) / len(diff_results)
+                
+                if successful:
+                    times = [r["elapsed_time"] for r in successful]
+                    metrics[f"{difficulty}_average_time"] = sum(times) / len(times)
+        
+        return metrics
+    
+    def compare_modes(self, agentic_metrics: Dict[str, Any], non_agentic_metrics: Dict[str, Any]) -> Dict[str, Any]:
+        """Compare agentic vs non-agentic performance"""
+        comparison = {}
+        
+        # Success rate comparison
+        comparison["success_rate_diff"] = (agentic_metrics.get("successful_cases", 0) / agentic_metrics["total_cases"] - 
+                                          non_agentic_metrics.get("successful_cases", 0) / non_agentic_metrics["total_cases"])
+        
+        # Time comparison
+        if "average_time" in agentic_metrics and "average_time" in non_agentic_metrics:
+            comparison["time_ratio"] = agentic_metrics["average_time"] / non_agentic_metrics["average_time"]
+            comparison["time_difference"] = agentic_metrics["average_time"] - non_agentic_metrics["average_time"]
+        
+        # Citation comparison
+        if "cases_with_citations" in agentic_metrics and "cases_with_citations" in non_agentic_metrics:
+            comparison["citation_rate_diff"] = (agentic_metrics["cases_with_citations"] / agentic_metrics["successful_cases"] - 
+                                               non_agentic_metrics["cases_with_citations"] / non_agentic_metrics["successful_cases"])
+        
+        # Response quality comparison
+        if "average_keyword_recall" in agentic_metrics and "average_keyword_recall" in non_agentic_metrics:
+            comparison["keyword_recall_improvement"] = (agentic_metrics["average_keyword_recall"] - 
+                                                       non_agentic_metrics["average_keyword_recall"])
+        
+        if "average_analysis_recall" in agentic_metrics and "average_analysis_recall" in non_agentic_metrics:
+            comparison["analysis_recall_improvement"] = (agentic_metrics["average_analysis_recall"] - 
+                                                        non_agentic_metrics["average_analysis_recall"])
+        
+        # Difficulty-specific comparison
+        for difficulty in ["easy", "medium", "hard"]:
+            key = f"{difficulty}_success_rate"
+            if key in agentic_metrics and key in non_agentic_metrics:
+                comparison[f"{difficulty}_success_improvement"] = (agentic_metrics[key] - non_agentic_metrics[key])
+        
+        return comparison
+    
+    def print_summary(self, results: Dict[str, Any]):
+        """Print evaluation summary"""
+        print("\n" + "=" * 80)
+        print("EVALUATION SUMMARY")
+        print("=" * 80)
+        
+        print(f"\nConfiguration:")
+        print(f"  LLM Provider: {results['config']['llm_provider']}")
+        print(f"  LLM Model: {results['config']['llm_model']}")
+        print(f"  Knowledge Base: {results['config']['kb_type']}")
+        
+        print(f"\n{'='*40} AGENTIC MODE {'='*40}")
+        self._print_mode_summary(results["agentic"]["metrics"])
+        
+        print(f"\n{'='*40} NON-AGENTIC MODE {'='*40}")
+        self._print_mode_summary(results["non_agentic"]["metrics"])
+        
+        print(f"\n{'='*40} COMPARISON {'='*40}")
+        comparison = results["comparison"]
+        
+        print(f"Success Rate Difference: {comparison.get('success_rate_diff', 0):.2%} (Agentic better)")
+        
+        if "time_ratio" in comparison:
+            print(f"Time Ratio: {comparison['time_ratio']:.2f}x (Agentic/Non-Agentic)")
+            print(f"Time Difference: {comparison['time_difference']:.2f} seconds")
+        
+        if "keyword_recall_improvement" in comparison:
+            print(f"Keyword Recall Improvement: {comparison['keyword_recall_improvement']:.2%}")
+        
+        if "analysis_recall_improvement" in comparison:
+            print(f"Analysis Recall Improvement: {comparison['analysis_recall_improvement']:.2%}")
+        
+        print("\nDifficulty-Specific Improvements:")
+        for difficulty in ["easy", "medium", "hard"]:
+            key = f"{difficulty}_success_improvement"
+            if key in comparison:
+                print(f"  {difficulty.capitalize()}: {comparison[key]:.2%}")
+        
+        print("=" * 80)
+    
+    def _print_mode_summary(self, metrics: Dict[str, Any]):
+        """Print summary for a single mode"""
+        print(f"Total Cases: {metrics['total_cases']}")
+        print(f"Successful: {metrics['successful_cases']} ({metrics['successful_cases']/metrics['total_cases']:.1%})")
+        print(f"Failed: {metrics['failed_cases']}")
+        
+        if "average_time" in metrics:
+            print(f"Average Time: {metrics['average_time']:.2f} seconds")
+            print(f"Total Time: {metrics['total_time']:.2f} seconds")
+        
+        if "cases_with_citations" in metrics:
+            print(f"Cases with Citations: {metrics['cases_with_citations']} ({metrics['cases_with_citations']/metrics['successful_cases']:.1%})")
+        
+        if "average_keyword_recall" in metrics:
+            print(f"Average Keyword Recall: {metrics['average_keyword_recall']:.2%}")
+        
+        if "average_analysis_recall" in metrics:
+            print(f"Average Analysis Recall: {metrics['average_analysis_recall']:.2%}")
+        
+        # Difficulty breakdown
+        print("\nBy Difficulty:")
+        for difficulty in ["easy", "medium", "hard"]:
+            success_key = f"{difficulty}_success_rate"
+            time_key = f"{difficulty}_average_time"
+            if success_key in metrics:
+                print(f"  {difficulty.capitalize()}: {metrics[success_key]:.1%} success", end="")
+                if time_key in metrics:
+                    print(f", {metrics[time_key]:.2f}s avg", end="")
+                print()
+
+
+def main():
+    """Main evaluation function"""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Evaluate Agentic RAG System")
+    parser.add_argument("--dataset", type=str, default="legal_qa_dataset.json",
+                       help="Path to evaluation dataset")
+    parser.add_argument("--output", type=str, default="results",
+                       help="Output directory for results")
+    parser.add_argument("--provider", type=str, help="Override LLM provider")
+    parser.add_argument("--model", type=str, help="Override LLM model")
+    parser.add_argument("--kb-type", choices=["local", "dify"], help="Knowledge base type")
+    
+    args = parser.parse_args()
+    
+    # Configure
+    config = Config.from_env()
+    if args.provider:
+        config.llm.provider = args.provider
+    if args.model:
+        config.llm.model = args.model
+    if args.kb_type:
+        from config import KnowledgeBaseType
+        config.knowledge_base.type = KnowledgeBaseType(args.kb_type)
+    
+    # Run evaluation
+    evaluator = RAGEvaluator(config)
+    results = evaluator.run_evaluation(args.dataset, args.output)
+    
+    return results
+
+
+if __name__ == "__main__":
+    main()

chapter3/contextual-retrieval/evaluation/retrieval_eval.json

@@ -0,0 +1,96 @@
+{
+  "description": "上下文感知检索 vs. 传统分块的检索召回评测集（对应实验 3-10）。gold_chunk_id 为人工标注的相关文本块，标注依据见 note 字段。语料为 document_store.json 中已建好索引的《宪法》与《检察官法》分块。",
+  "corpus": "document_store.json",
+  "queries": [
+    {
+      "id": "q01",
+      "query": "宪法是哪一年通过的？后来经过了几次修正？",
+      "gold_chunk_id": "宪法_chunk_0",
+      "note": "chunk_0 为宪法开篇，列举 1982 年通过日期及历次修正案日期。"
+    },
+    {
+      "id": "q02",
+      "query": "国家的根本任务是什么？",
+      "gold_chunk_id": "宪法_chunk_1",
+      "note": "chunk_1 序言明确写明“国家的根本任务是……”。"
+    },
+    {
+      "id": "q03",
+      "query": "我国处理民族关系的基本原则和对外政策中的五项原则是什么？",
+      "gold_chunk_id": "宪法_chunk_2",
+      "note": "chunk_2 阐述社会主义民族关系与和平共处五项原则。"
+    },
+    {
+      "id": "q04",
+      "query": "少数民族聚居的地方如何实行区域自治？",
+      "gold_chunk_id": "宪法_chunk_3",
+      "note": "chunk_3 第四条前后规定民族区域自治与依法治国。"
+    },
+    {
+      "id": "q05",
+      "query": "宪法如何保护个体经济、私营经济等非公有制经济？",
+      "gold_chunk_id": "宪法_chunk_4",
+      "note": "chunk_4 含第十一至十三条，规定非公有制经济与私有财产保护。"
+    },
+    {
+      "id": "q06",
+      "query": "宪法关于设立特别行政区以及在华外国人权利的规定是什么？",
+      "gold_chunk_id": "宪法_chunk_6",
+      "note": "chunk_6 含第三十一、三十二条，特别行政区与外国人条款。"
+    },
+    {
+      "id": "q07",
+      "query": "公民对国家机关工作人员的违法失职行为享有哪些监督权利？",
+      "gold_chunk_id": "宪法_chunk_7",
+      "note": "chunk_7 第四十一条规定批评、建议、申诉、控告、检举权。"
+    },
+    {
+      "id": "q08",
+      "query": "全国人民代表大会常务委员会有哪些职权？",
+      "gold_chunk_id": "宪法_chunk_10",
+      "note": "旗舰对照案例：chunk_10 原文以“（四）解释法律……”开头，正文既无“常务委员会”也无“职权”字样，纯靠上下文前缀（“第六十七条 全国人民代表大会常务委员会行使下列职权”）才能被锚定；无上下文时极易被 chunk_9/chunk_11 抢占。"
+    },
+    {
+      "id": "q09",
+      "query": "国家主席有哪些职权？",
+      "gold_chunk_id": "宪法_chunk_12",
+      "note": "chunk_12 含第八十至八十二条，国家主席公布法律、任免国务院总理等职权。"
+    },
+    {
+      "id": "q10",
+      "query": "地方各级人民代表大会的任期和代表选举方式是怎样规定的？",
+      "gold_chunk_id": "宪法_chunk_14",
+      "note": "chunk_14 第九十七至九十九条规定地方人大代表选举与五年任期。"
+    },
+    {
+      "id": "q11",
+      "query": "县级以上地方各级人民代表大会常务委员会组成人员有什么兼职限制？",
+      "gold_chunk_id": "宪法_chunk_15",
+      "note": "chunk_15 规定常委会组成人员不得担任行政、监察、审判、检察机关职务。"
+    },
+    {
+      "id": "q12",
+      "query": "监察委员会的性质、组成和任期是怎样规定的？",
+      "gold_chunk_id": "宪法_chunk_17",
+      "note": "chunk_17 第七节监察委员会，规定其性质、组成与任期。"
+    },
+    {
+      "id": "q13",
+      "query": "检察官法是什么时候修订的？",
+      "gold_chunk_id": "检察官法_2019_04_23_chunk_0",
+      "note": "chunk_0 列举检察官法 1995 年通过及历次修正、2019 年修订日期。"
+    },
+    {
+      "id": "q14",
+      "query": "检察官遴选委员会由哪些人员组成？律师参加公开选拔需要什么条件？",
+      "gold_chunk_id": "检察官法_2019_04_23_chunk_2",
+      "note": "chunk_2 第十六条规定省级检察官遴选委员会组成及律师公开选拔条件。"
+    },
+    {
+      "id": "q15",
+      "query": "检察官在哪些情形下应当予以免职？",
+      "gold_chunk_id": "检察官法_2019_04_23_chunk_3",
+      "note": "chunk_3 列举检察官免职情形及违反条件任命的撤销程序。"
+    }
+  ]
+}