ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter3/agentic-rag/evaluation/dataset_builder.py

@@ -0,0 +1,317 @@
+"""Build evaluation dataset from Chinese legal documents"""
+
+import json
+import logging
+from typing import List, Dict, Any
+from pathlib import Path
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class LegalDatasetBuilder:
+    """Build evaluation dataset for Chinese legal Q&A"""
+    
+    def __init__(self):
+        self.simple_cases = []
+        self.complex_cases = []
+        
+    def create_simple_cases(self) -> List[Dict[str, Any]]:
+        """Create simple direct legal questions"""
+        simple_cases = [
+            {
+                "id": "simple_1",
+                "question": "故意杀人罪判几年？",
+                "expected_keywords": ["死刑", "无期徒刑", "十年以上有期徒刑"],
+                "reference": "《中华人民共和国刑法》第二百三十二条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_2",
+                "question": "盗窃罪的立案标准是什么？",
+                "expected_keywords": ["一千元", "三千元", "数额较大"],
+                "reference": "《中华人民共和国刑法》第二百六十四条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_3", 
+                "question": "醉酒驾驶机动车如何处罚？",
+                "expected_keywords": ["拘役", "罚金", "吊销驾照"],
+                "reference": "《中华人民共和国刑法》第一百三十三条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_4",
+                "question": "诈骗罪的量刑标准是什么？",
+                "expected_keywords": ["三年以下", "三年以上十年以下", "十年以上"],
+                "reference": "《中华人民共和国刑法》第二百六十六条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_5",
+                "question": "故意伤害罪致人重伤的处罚是什么？",
+                "expected_keywords": ["三年以上十年以下", "有期徒刑"],
+                "reference": "《中华人民共和国刑法》第二百三十四条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_6",
+                "question": "抢劫罪的加重情节有哪些？",
+                "expected_keywords": ["入户抢劫", "多次抢劫", "抢劫数额巨大"],
+                "reference": "《中华人民共和国刑法》第二百六十三条",
+                "difficulty": "medium"
+            },
+            {
+                "id": "simple_7",
+                "question": "非法拘禁罪的构成要件是什么？",
+                "expected_keywords": ["非法", "拘禁", "限制人身自由"],
+                "reference": "《中华人民共和国刑法》第二百三十八条",
+                "difficulty": "medium"
+            },
+            {
+                "id": "simple_8",
+                "question": "贪污罪的数额标准如何认定？",
+                "expected_keywords": ["三万元", "二十万元", "三百万元"],
+                "reference": "《中华人民共和国刑法》第三百八十三条",
+                "difficulty": "medium"
+            },
+            {
+                "id": "simple_9",
+                "question": "交通肇事罪的立案标准是什么？",
+                "expected_keywords": ["死亡一人", "重伤三人", "财产损失"],
+                "reference": "《中华人民共和国刑法》第一百三十三条",
+                "difficulty": "easy"
+            },
+            {
+                "id": "simple_10",
+                "question": "寻衅滋事罪如何处罚？",
+                "expected_keywords": ["五年以下", "有期徒刑", "拘役", "管制"],
+                "reference": "《中华人民共和国刑法》第二百九十三条",
+                "difficulty": "easy"
+            }
+        ]
+        
+        return simple_cases
+    
+    def create_complex_cases(self) -> List[Dict[str, Any]]:
+        """Create complex legal scenario questions"""
+        complex_cases = [
+            {
+                "id": "complex_1",
+                "question": """张某因与李某发生经济纠纷，持刀闯入李某家中，意图讨债。在争执过程中，张某用刀刺伤李某，
+                            导致李某重伤。同时，张某还顺手拿走了李某家中的现金5万元。请问张某的行为应如何定性？
+                            可能面临什么样的刑事处罚？""",
+                "expected_analysis": ["入户抢劫", "故意伤害", "数罪并罚"],
+                "reference": "《刑法》第二百三十四条、第二百六十三条",
+                "difficulty": "hard",
+                "requires_multi_query": True
+            },
+            {
+                "id": "complex_2",
+                "question": """王某系某国有企业财务主管，利用职务之便，通过虚开发票等手段，
+                            将公司资金200万元转入其控制的账户。后王某用该资金进行股票投资，
+                            获利50万元。案发后，王某主动退还全部赃款。请分析王某的法律责任。""",
+                "expected_analysis": ["贪污罪", "挪用公款罪", "自首情节", "退赃"],
+                "reference": "《刑法》第三百八十二条、第三百八十四条",
+                "difficulty": "hard",
+                "requires_multi_query": True
+            },
+            {
+                "id": "complex_3",
+                "question": """赵某酒后驾车，在市区超速行驶，撞倒正在过马路的行人陈某，
+                            导致陈某当场死亡。赵某见状，驾车逃离现场。第二天，在家人劝说下，
+                            赵某到公安机关投案自首。请问赵某涉嫌哪些犯罪？量刑时应考虑哪些因素？""",
+                "expected_analysis": ["交通肇事罪", "危险驾驶罪", "逃逸", "自首"],
+                "reference": "《刑法》第一百三十三条",
+                "difficulty": "hard",
+                "requires_multi_query": True
+            },
+            {
+                "id": "complex_4",
+                "question": """刘某通过网络平台发布虚假投资信息，声称可以保证高额回报，
+                            先后骗取30名投资者共计500万元。其中，刘某将200万元用于个人挥霍，
+                            300万元用于归还之前的债务。请问刘某的行为如何定性？可能的量刑是什么？""",
+                "expected_analysis": ["诈骗罪", "数额特别巨大", "多人受害"],
+                "reference": "《刑法》第二百六十六条",
+                "difficulty": "hard",
+                "requires_multi_query": True
+            },
+            {
+                "id": "complex_5",
+                "question": """孙某与钱某共谋盗窃某商场。孙某负责望风，钱某进入商场实施盗窃。
+                            钱某在盗窃过程中被保安发现，为逃跑将保安打成轻伤。
+                            最终二人盗窃财物价值8万元。请分析孙某和钱某各自的刑事责任。""",
+                "expected_analysis": ["共同犯罪", "盗窃罪", "抢劫罪", "转化犯"],
+                "reference": "《刑法》第二百六十四条、第二百六十九条",
+                "difficulty": "hard",
+                "requires_multi_query": True
+            }
+        ]
+        
+        return complex_cases
+    
+    def build_dataset(self, output_path: str = "legal_qa_dataset.json"):
+        """Build and save the complete dataset"""
+        dataset = {
+            "simple_cases": self.create_simple_cases(),
+            "complex_cases": self.create_complex_cases(),
+            "metadata": {
+                "total_cases": 15,
+                "simple_count": 10,
+                "complex_count": 5,
+                "domain": "Chinese Criminal Law",
+                "purpose": "Evaluate agentic vs non-agentic RAG performance"
+            }
+        }
+        
+        # Save dataset
+        with open(output_path, 'w', encoding='utf-8') as f:
+            json.dump(dataset, f, ensure_ascii=False, indent=2)
+        
+        logger.info(f"Dataset saved to {output_path}")
+        return dataset
+
+
+def create_legal_documents() -> List[Dict[str, str]]:
+    """Create sample legal documents for the knowledge base"""
+    documents = [
+        {
+            "doc_id": "criminal_law_homicide",
+            "title": "刑法-故意杀人罪",
+            "content": """第二百三十二条 【故意杀人罪】故意杀人的，处死刑、无期徒刑或者十年以上有期徒刑；
+            情节较轻的，处三年以上十年以下有期徒刑。
+            
+            故意杀人罪是指故意非法剥夺他人生命的行为。该罪侵犯的客体是他人的生命权。
+            法律依据是《中华人民共和国刑法》第二百三十二条。
+            
+            量刑标准：
+            1. 情节严重的：死刑、无期徒刑或十年以上有期徒刑
+            2. 情节较轻的：三年以上十年以下有期徒刑
+            
+            情节较轻通常包括：防卫过当、义愤杀人、被害人有过错等情形。"""
+        },
+        {
+            "doc_id": "criminal_law_theft",
+            "title": "刑法-盗窃罪",
+            "content": """第二百六十四条 【盗窃罪】盗窃公私财物，数额较大的，或者多次盗窃、入户盗窃、
+            携带凶器盗窃、扒窃的，处三年以下有期徒刑、拘役或者管制，并处或者单处罚金；
+            数额巨大或者有其他严重情节的，处三年以上十年以下有期徒刑，并处罚金；
+            数额特别巨大或者有其他特别严重情节的，处十年以上有期徒刑或者无期徒刑，并处罚金或者没收财产。
+            
+            盗窃罪的立案标准：
+            1. 数额较大：一般为1000元至3000元以上
+            2. 数额巨大：一般为3万元至10万元以上  
+            3. 数额特别巨大：一般为30万元至50万元以上
+            
+            特殊情形：多次盗窃（2年内3次以上）、入户盗窃、携带凶器盗窃、扒窃的，
+            不论数额大小，均构成盗窃罪。"""
+        },
+        {
+            "doc_id": "criminal_law_fraud",
+            "title": "刑法-诈骗罪",
+            "content": """第二百六十六条 【诈骗罪】诈骗公私财物，数额较大的，处三年以下有期徒刑、
+            拘役或者管制，并处或者单处罚金；数额巨大或者有其他严重情节的，
+            处三年以上十年以下有期徒刑，并处罚金；数额特别巨大或者有其他特别严重情节的，
+            处十年以上有期徒刑或者无期徒刑，并处罚金或者没收财产。
+            
+            诈骗罪的量刑标准：
+            1. 数额较大（3千元至1万元以上）：三年以下有期徒刑、拘役或者管制
+            2. 数额巨大（3万元至10万元以上）：三年以上十年以下有期徒刑
+            3. 数额特别巨大（50万元以上）：十年以上有期徒刑或者无期徒刑
+            
+            诈骗罪是指以非法占有为目的，用虚构事实或者隐瞒真相的方法，
+            骗取数额较大的公私财物的行为。"""
+        },
+        {
+            "doc_id": "criminal_law_robbery",
+            "title": "刑法-抢劫罪",
+            "content": """第二百六十三条 【抢劫罪】以暴力、胁迫或者其他方法抢劫公私财物的，
+            处三年以上十年以下有期徒刑，并处罚金；有下列情形之一的，
+            处十年以上有期徒刑、无期徒刑或者死刑，并处罚金或者没收财产：
+            
+            （一）入户抢劫的；
+            （二）在公共交通工具上抢劫的；
+            （三）抢劫银行或者其他金融机构的；
+            （四）多次抢劫或者抢劫数额巨大的；
+            （五）抢劫致人重伤、死亡的；
+            （六）冒充军警人员抢劫的；
+            （七）持枪抢劫的；
+            （八）抢劫军用物资或者抢险、救灾、救济物资的。
+            
+            抢劫罪的加重处罚情节包括上述八种情形，有其中之一的，
+            最低刑期为十年有期徒刑。"""
+        },
+        {
+            "doc_id": "criminal_law_injury",
+            "title": "刑法-故意伤害罪",
+            "content": """第二百三十四条 【故意伤害罪】故意伤害他人身体的，处三年以下有期徒刑、
+            拘役或者管制。犯前款罪，致人重伤的，处三年以上十年以下有期徒刑；
+            致人死亡或者以特别残忍手段致人重伤造成严重残疾的，处十年以上有期徒刑、
+            无期徒刑或者死刑。
+            
+            故意伤害罪的量刑：
+            1. 故意伤害致人轻伤的：三年以下有期徒刑、拘役或者管制
+            2. 故意伤害致人重伤的：三年以上十年以下有期徒刑
+            3. 故意伤害致人死亡或特别残忍手段致残的：十年以上有期徒刑、无期徒刑或死刑
+            
+            重伤标准：使人肢体残废或者毁人容貌；使人丧失听觉、视觉或者其他器官功能；
+            其他对于人身健康有重大伤害的。"""
+        },
+        {
+            "doc_id": "criminal_law_traffic",
+            "title": "刑法-交通肇事罪与危险驾驶罪",
+            "content": """第一百三十三条 【交通肇事罪】违反交通运输管理法规，因而发生重大事故，
+            致人重伤、死亡或者使公私财产遭受重大损失的，处三年以下有期徒刑或者拘役；
+            交通运输肇事后逃逸或者有其他特别恶劣情节的，处三年以上七年以下有期徒刑；
+            因逃逸致人死亡的，处七年以上有期徒刑。
+            
+            第一百三十三条之一 【危险驾驶罪】在道路上驾驶机动车，有下列情形之一的，
+            处拘役，并处罚金：
+            （一）追逐竞驶，情节恶劣的；
+            （二）醉酒驾驶机动车的；
+            （三）从事校车业务或者旅客运输，严重超过额定乘员载客，
+                  或者严重超过规定时速行驶的；
+            （四）违反危险化学品安全管理规定运输危险化学品，危及公共安全的。
+            
+            醉酒驾驶的认定标准：血液酒精含量达到80毫克/100毫升以上。"""
+        },
+        {
+            "doc_id": "criminal_law_corruption",
+            "title": "刑法-贪污罪",
+            "content": """第三百八十二条 【贪污罪】国家工作人员利用职务上的便利，侵吞、窃取、
+            骗取或者以其他手段非法占有公共财物的，是贪污罪。
+            
+            第三百八十三条 【贪污罪的处罚规定】对犯贪污罪的，根据情节轻重，分别依照下列规定处罚：
+            
+            （一）贪污数额较大或者有其他较重情节的，处三年以下有期徒刑或者拘役，并处罚金。
+            （二）贪污数额巨大或者有其他严重情节的，处三年以上十年以下有期徒刑，并处罚金或者没收财产。
+            （三）贪污数额特别巨大或者有其他特别严重情节的，处十年以上有期徒刑或者无期徒刑，
+                  并处罚金或者没收财产；数额特别巨大，并使国家和人民利益遭受特别重大损失的，
+                  处无期徒刑或者死刑，并处没收财产。
+            
+            贪污数额标准：
+            1. 数额较大：三万元以上不满二十万元
+            2. 数额巨大：二十万元以上不满三百万元
+            3. 数额特别巨大：三百万元以上"""
+        }
+    ]
+    
+    return documents
+
+
+if __name__ == "__main__":
+    # Build evaluation dataset
+    builder = LegalDatasetBuilder()
+    dataset = builder.build_dataset("legal_qa_dataset.json")
+    
+    print(f"Dataset created with {len(dataset['simple_cases'])} simple cases and {len(dataset['complex_cases'])} complex cases")
+    
+    # Create legal documents
+    documents = create_legal_documents()
+    
+    # Save documents
+    with open("legal_documents.json", 'w', encoding='utf-8') as f:
+        json.dump(documents, f, ensure_ascii=False, indent=2)
+    
+    print(f"Created {len(documents)} legal documents for knowledge base")

chapter3/agentic-rag/evaluation/evaluate.py

@@ -0,0 +1,404 @@
+"""Evaluation framework for Agentic RAG system"""
+
+import json
+import logging
+import time
+from typing import List, Dict, Any, Optional
+from pathlib import Path
+import sys
+import os
+
+# ``evaluation`` is intentionally runnable both as a script directory and via
+# pytest from the repository root.  Put this experiment's directory first so
+# the unqualified educational imports below cannot resolve a sibling
+# experiment's ``config.py``/``agent.py`` from an earlier sys.path entry.
+_PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if _PROJECT_DIR in sys.path:
+    sys.path.remove(_PROJECT_DIR)
+sys.path.insert(0, _PROJECT_DIR)
+
+from config import Config
+from agent import AgenticRAG
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class RAGEvaluator:
+    """Evaluate RAG system performance"""
+    
+    def __init__(self, config: Optional[Config] = None):
+        self.config = config or Config.from_env()
+        self.agent = AgenticRAG(self.config)
+        self.results = {
+            "agentic": [],
+            "non_agentic": []
+        }
+        
+    def load_dataset(self, dataset_path: str) -> Dict[str, Any]:
+        """Load evaluation dataset"""
+        with open(dataset_path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    
+    def evaluate_response(self, 
+                         response: str, 
+                         test_case: Dict[str, Any]) -> Dict[str, Any]:
+        """Evaluate a single response"""
+        evaluation = {
+            "case_id": test_case["id"],
+            "question": test_case["question"],
+            "response": response,
+            "metrics": {}
+        }
+        
+        # Check for expected keywords (for simple cases)
+        if "expected_keywords" in test_case:
+            keywords_found = []
+            keywords_missing = []
+            
+            for keyword in test_case["expected_keywords"]:
+                if keyword.lower() in response.lower():
+                    keywords_found.append(keyword)
+                else:
+                    keywords_missing.append(keyword)
+            
+            evaluation["metrics"]["keyword_recall"] = len(keywords_found) / len(test_case["expected_keywords"]) if test_case["expected_keywords"] else 1.0
+            evaluation["metrics"]["keywords_found"] = keywords_found
+            evaluation["metrics"]["keywords_missing"] = keywords_missing
+        
+        # Check for analysis points (for complex cases)
+        if "expected_analysis" in test_case:
+            analysis_found = []
+            analysis_missing = []
+            
+            for point in test_case["expected_analysis"]:
+                if point.lower() in response.lower():
+                    analysis_found.append(point)
+                else:
+                    analysis_missing.append(point)
+            
+            evaluation["metrics"]["analysis_recall"] = len(analysis_found) / len(test_case["expected_analysis"]) if test_case["expected_analysis"] else 1.0
+            evaluation["metrics"]["analysis_found"] = analysis_found
+            evaluation["metrics"]["analysis_missing"] = analysis_missing
+        
+        # Check for citations
+        citation_count = response.count("[Doc:") + response.count("[Chunk:")
+        evaluation["metrics"]["has_citations"] = citation_count > 0
+        evaluation["metrics"]["citation_count"] = citation_count
+        
+        # Response length
+        evaluation["metrics"]["response_length"] = len(response)
+        
+        # Check if response indicates no answer
+        no_answer_indicators = ["无法回答", "没有找到", "知识库中没有", "cannot answer", "not found"]
+        evaluation["metrics"]["gave_answer"] = not any(indicator in response.lower() for indicator in no_answer_indicators)
+        
+        return evaluation
+    
+    def run_test_case(self, test_case: Dict[str, Any], mode: str = "agentic") -> Dict[str, Any]:
+        """Run a single test case"""
+        logger.info(f"Running {mode} mode for case {test_case['id']}")
+        
+        start_time = time.time()
+        
+        try:
+            if mode == "agentic":
+                response = self.agent.query(test_case["question"], stream=False)
+            else:
+                response = self.agent.query_non_agentic(test_case["question"], stream=False)
+            
+            elapsed_time = time.time() - start_time
+            
+            # Clear history for next test
+            self.agent.clear_history()
+            
+            # Evaluate response
+            evaluation = self.evaluate_response(response, test_case)
+            evaluation["mode"] = mode
+            evaluation["elapsed_time"] = elapsed_time
+            evaluation["difficulty"] = test_case.get("difficulty", "unknown")
+            evaluation["success"] = True
+            
+        except Exception as e:
+            logger.error(f"Error in test case {test_case['id']}: {e}")
+            evaluation = {
+                "case_id": test_case["id"],
+                "question": test_case["question"],
+                "mode": mode,
+                "success": False,
+                "error": str(e),
+                "elapsed_time": time.time() - start_time
+            }
+        
+        return evaluation
+    
+    def run_evaluation(self, dataset_path: str, output_dir: str = "results"):
+        """Run full evaluation"""
+        # Load dataset
+        dataset = self.load_dataset(dataset_path)
+        
+        # Create output directory
+        output_path = Path(output_dir)
+        output_path.mkdir(exist_ok=True)
+        
+        # Combine all test cases
+        all_cases = dataset["simple_cases"] + dataset["complex_cases"]
+        
+        # Run agentic mode
+        logger.info("=" * 60)
+        logger.info("Running AGENTIC mode evaluation")
+        logger.info("=" * 60)
+        
+        agentic_results = []
+        for test_case in all_cases:
+            result = self.run_test_case(test_case, mode="agentic")
+            agentic_results.append(result)
+            time.sleep(1)  # Rate limiting
+        
+        # Run non-agentic mode
+        logger.info("=" * 60)
+        logger.info("Running NON-AGENTIC mode evaluation")
+        logger.info("=" * 60)
+        
+        non_agentic_results = []
+        for test_case in all_cases:
+            result = self.run_test_case(test_case, mode="non_agentic")
+            non_agentic_results.append(result)
+            time.sleep(1)  # Rate limiting
+        
+        # Compute aggregate metrics
+        agentic_metrics = self.compute_aggregate_metrics(agentic_results)
+        non_agentic_metrics = self.compute_aggregate_metrics(non_agentic_results)
+        
+        # Save results
+        results = {
+            "dataset": dataset_path,
+            "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
+            "config": {
+                "llm_provider": self.config.llm.provider,
+                "llm_model": self.agent.model,
+                "kb_type": self.config.knowledge_base.type.value
+            },
+            "agentic": {
+                "results": agentic_results,
+                "metrics": agentic_metrics
+            },
+            "non_agentic": {
+                "results": non_agentic_results,
+                "metrics": non_agentic_metrics
+            },
+            "comparison": self.compare_modes(agentic_metrics, non_agentic_metrics)
+        }
+        
+        # Save to file
+        output_file = output_path / f"evaluation_results_{time.strftime('%Y%m%d_%H%M%S')}.json"
+        with open(output_file, 'w', encoding='utf-8') as f:
+            json.dump(results, f, ensure_ascii=False, indent=2)
+        
+        logger.info(f"Results saved to {output_file}")
+        
+        # Print summary
+        self.print_summary(results)
+        
+        return results
+    
+    def compute_aggregate_metrics(self, results: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Compute aggregate metrics from results"""
+        metrics = {
+            "total_cases": len(results),
+            "successful_cases": sum(1 for r in results if r.get("success", False)),
+            "failed_cases": sum(1 for r in results if not r.get("success", False)),
+            "average_time": 0,
+            "total_time": 0
+        }
+        
+        # Separate by difficulty
+        simple_results = [r for r in results if r.get("difficulty") == "easy"]
+        medium_results = [r for r in results if r.get("difficulty") == "medium"]
+        hard_results = [r for r in results if r.get("difficulty") == "hard"]
+        
+        # Compute metrics for successful cases
+        successful_results = [r for r in results if r.get("success", False)]
+        
+        if successful_results:
+            # Time metrics
+            times = [r["elapsed_time"] for r in successful_results]
+            metrics["average_time"] = sum(times) / len(times)
+            metrics["total_time"] = sum(times)
+            metrics["min_time"] = min(times)
+            metrics["max_time"] = max(times)
+            
+            # Response quality metrics
+            metrics["cases_with_citations"] = sum(1 for r in successful_results 
+                                                 if r.get("metrics", {}).get("has_citations", False))
+            metrics["cases_gave_answer"] = sum(1 for r in successful_results 
+                                              if r.get("metrics", {}).get("gave_answer", False))
+            
+            # Average response length
+            lengths = [r.get("metrics", {}).get("response_length", 0) for r in successful_results]
+            metrics["average_response_length"] = sum(lengths) / len(lengths) if lengths else 0
+            
+            # Keyword/analysis recall (for cases that have them)
+            keyword_recalls = [r["metrics"]["keyword_recall"] for r in successful_results 
+                              if "keyword_recall" in r.get("metrics", {})]
+            if keyword_recalls:
+                metrics["average_keyword_recall"] = sum(keyword_recalls) / len(keyword_recalls)
+            
+            analysis_recalls = [r["metrics"]["analysis_recall"] for r in successful_results 
+                               if "analysis_recall" in r.get("metrics", {})]
+            if analysis_recalls:
+                metrics["average_analysis_recall"] = sum(analysis_recalls) / len(analysis_recalls)
+        
+        # Metrics by difficulty
+        for difficulty, diff_results in [("easy", simple_results), ("medium", medium_results), ("hard", hard_results)]:
+            if diff_results:
+                successful = [r for r in diff_results if r.get("success", False)]
+                metrics[f"{difficulty}_success_rate"] = len(successful) / len(diff_results)
+                
+                if successful:
+                    times = [r["elapsed_time"] for r in successful]
+                    metrics[f"{difficulty}_average_time"] = sum(times) / len(times)
+        
+        return metrics
+    
+    def compare_modes(self, agentic_metrics: Dict[str, Any], non_agentic_metrics: Dict[str, Any]) -> Dict[str, Any]:
+        """Compare agentic vs non-agentic performance"""
+        comparison = {}
+        
+        # Success rate comparison
+        comparison["success_rate_diff"] = (agentic_metrics.get("successful_cases", 0) / agentic_metrics["total_cases"] - 
+                                          non_agentic_metrics.get("successful_cases", 0) / non_agentic_metrics["total_cases"])
+        
+        # Time comparison
+        if "average_time" in agentic_metrics and "average_time" in non_agentic_metrics:
+            comparison["time_ratio"] = agentic_metrics["average_time"] / non_agentic_metrics["average_time"]
+            comparison["time_difference"] = agentic_metrics["average_time"] - non_agentic_metrics["average_time"]
+        
+        # Citation comparison
+        if "cases_with_citations" in agentic_metrics and "cases_with_citations" in non_agentic_metrics:
+            comparison["citation_rate_diff"] = (agentic_metrics["cases_with_citations"] / agentic_metrics["successful_cases"] - 
+                                               non_agentic_metrics["cases_with_citations"] / non_agentic_metrics["successful_cases"])
+        
+        # Response quality comparison
+        if "average_keyword_recall" in agentic_metrics and "average_keyword_recall" in non_agentic_metrics:
+            comparison["keyword_recall_improvement"] = (agentic_metrics["average_keyword_recall"] - 
+                                                       non_agentic_metrics["average_keyword_recall"])
+        
+        if "average_analysis_recall" in agentic_metrics and "average_analysis_recall" in non_agentic_metrics:
+            comparison["analysis_recall_improvement"] = (agentic_metrics["average_analysis_recall"] - 
+                                                        non_agentic_metrics["average_analysis_recall"])
+        
+        # Difficulty-specific comparison
+        for difficulty in ["easy", "medium", "hard"]:
+            key = f"{difficulty}_success_rate"
+            if key in agentic_metrics and key in non_agentic_metrics:
+                comparison[f"{difficulty}_success_improvement"] = (agentic_metrics[key] - non_agentic_metrics[key])
+        
+        return comparison
+    
+    def print_summary(self, results: Dict[str, Any]):
+        """Print evaluation summary"""
+        print("\n" + "=" * 80)
+        print("EVALUATION SUMMARY")
+        print("=" * 80)
+        
+        print(f"\nConfiguration:")
+        print(f"  LLM Provider: {results['config']['llm_provider']}")
+        print(f"  LLM Model: {results['config']['llm_model']}")
+        print(f"  Knowledge Base: {results['config']['kb_type']}")
+        
+        print(f"\n{'='*40} AGENTIC MODE {'='*40}")
+        self._print_mode_summary(results["agentic"]["metrics"])
+        
+        print(f"\n{'='*40} NON-AGENTIC MODE {'='*40}")
+        self._print_mode_summary(results["non_agentic"]["metrics"])
+        
+        print(f"\n{'='*40} COMPARISON {'='*40}")
+        comparison = results["comparison"]
+        
+        print(f"Success Rate Difference: {comparison.get('success_rate_diff', 0):.2%} (Agentic better)")
+        
+        if "time_ratio" in comparison:
+            print(f"Time Ratio: {comparison['time_ratio']:.2f}x (Agentic/Non-Agentic)")
+            print(f"Time Difference: {comparison['time_difference']:.2f} seconds")
+        
+        if "keyword_recall_improvement" in comparison:
+            print(f"Keyword Recall Improvement: {comparison['keyword_recall_improvement']:.2%}")
+        
+        if "analysis_recall_improvement" in comparison:
+            print(f"Analysis Recall Improvement: {comparison['analysis_recall_improvement']:.2%}")
+        
+        print("\nDifficulty-Specific Improvements:")
+        for difficulty in ["easy", "medium", "hard"]:
+            key = f"{difficulty}_success_improvement"
+            if key in comparison:
+                print(f"  {difficulty.capitalize()}: {comparison[key]:.2%}")
+        
+        print("=" * 80)
+    
+    def _print_mode_summary(self, metrics: Dict[str, Any]):
+        """Print summary for a single mode"""
+        print(f"Total Cases: {metrics['total_cases']}")
+        print(f"Successful: {metrics['successful_cases']} ({metrics['successful_cases']/metrics['total_cases']:.1%})")
+        print(f"Failed: {metrics['failed_cases']}")
+        
+        if "average_time" in metrics:
+            print(f"Average Time: {metrics['average_time']:.2f} seconds")
+            print(f"Total Time: {metrics['total_time']:.2f} seconds")
+        
+        if "cases_with_citations" in metrics:
+            print(f"Cases with Citations: {metrics['cases_with_citations']} ({metrics['cases_with_citations']/metrics['successful_cases']:.1%})")
+        
+        if "average_keyword_recall" in metrics:
+            print(f"Average Keyword Recall: {metrics['average_keyword_recall']:.2%}")
+        
+        if "average_analysis_recall" in metrics:
+            print(f"Average Analysis Recall: {metrics['average_analysis_recall']:.2%}")
+        
+        # Difficulty breakdown
+        print("\nBy Difficulty:")
+        for difficulty in ["easy", "medium", "hard"]:
+            success_key = f"{difficulty}_success_rate"
+            time_key = f"{difficulty}_average_time"
+            if success_key in metrics:
+                print(f"  {difficulty.capitalize()}: {metrics[success_key]:.1%} success", end="")
+                if time_key in metrics:
+                    print(f", {metrics[time_key]:.2f}s avg", end="")
+                print()
+
+
+def main():
+    """Main evaluation function"""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Evaluate Agentic RAG System")
+    parser.add_argument("--dataset", type=str, default="legal_qa_dataset.json",
+                       help="Path to evaluation dataset")
+    parser.add_argument("--output", type=str, default="results",
+                       help="Output directory for results")
+    parser.add_argument("--provider", type=str, help="Override LLM provider")
+    parser.add_argument("--model", type=str, help="Override LLM model")
+    parser.add_argument("--kb-type", choices=["local", "dify"], help="Knowledge base type")
+    
+    args = parser.parse_args()
+    
+    # Configure
+    config = Config.from_env()
+    if args.provider:
+        config.llm.provider = args.provider
+    if args.model:
+        config.llm.model = args.model
+    if args.kb_type:
+        from config import KnowledgeBaseType
+        config.knowledge_base.type = KnowledgeBaseType(args.kb_type)
+    
+    # Run evaluation
+    evaluator = RAGEvaluator(config)
+    results = evaluator.run_evaluation(args.dataset, args.output)
+    
+    return results
+
+
+if __name__ == "__main__":
+    main()

chapter3/agentic-rag/evaluation/offline_qa.json

@@ -0,0 +1,66 @@
+{
+  "description": "离线检索对比数据集：用于在无需 LLM / 无需外部检索服务的情况下，量化对比『非智能体化 RAG（单次检索）』与『智能体化 RAG（多轮/分解检索）』的证据召回能力。每条问题标注了回答所必需的金标准法条（gold_articles，以法条编号精确匹配）。naive_query 为用户原始提问（单次检索直接使用）；subqueries 为智能体经过思考后分解/改写出的检索式（多次检索后取并集）。金标准法条均已确认存在于 laws/ 语料中。",
+  "metric": "证据召回率 = 命中的金标准法条数 / 金标准法条总数（某法条被命中当且仅当检索结果中存在以该法条编号开头的分块）",
+  "cases": [
+    {
+      "id": "easy_1",
+      "question": "故意伤害致人重伤的，如何处罚？",
+      "difficulty": "easy",
+      "naive_query": "故意伤害致人重伤的，如何处罚？",
+      "subqueries": ["故意伤害 致人重伤 有期徒刑"],
+      "gold_articles": ["第二百三十四条"]
+    },
+    {
+      "id": "easy_2",
+      "question": "正当防卫是怎么规定的？",
+      "difficulty": "easy",
+      "naive_query": "正当防卫是怎么规定的？",
+      "subqueries": ["正当防卫 不负刑事责任"],
+      "gold_articles": ["第二十条"]
+    },
+    {
+      "id": "easy_3",
+      "question": "醉酒驾驶机动车如何处罚？",
+      "difficulty": "easy",
+      "naive_query": "醉酒驾驶机动车如何处罚？",
+      "subqueries": ["危险驾驶罪 醉酒 驾驶机动车 拘役"],
+      "gold_articles": ["第一百三十三条之一"]
+    },
+    {
+      "id": "hard_1",
+      "question": "故意杀人罪判几年？",
+      "difficulty": "hard",
+      "naive_query": "故意杀人罪判几年？",
+      "subqueries": ["故意杀人 死刑 无期徒刑"],
+      "gold_articles": ["第二百三十二条"]
+    },
+    {
+      "id": "hard_2",
+      "question": "盗窃罪的立案标准是什么？",
+      "difficulty": "hard",
+      "naive_query": "盗窃罪的立案标准是什么？",
+      "subqueries": ["盗窃 公私财物 数额较大 有期徒刑"],
+      "gold_articles": ["第二百六十四条"]
+    },
+    {
+      "id": "hard_3",
+      "question": "诈骗罪的量刑标准是什么？",
+      "difficulty": "hard",
+      "naive_query": "诈骗罪的量刑标准是什么？",
+      "subqueries": ["诈骗 公私财物 数额较大 有期徒刑 罚金"],
+      "gold_articles": ["第二百六十六条"]
+    },
+    {
+      "id": "hard_4",
+      "question": "醉酒过失致人重伤且有盗窃前科，应如何量刑？",
+      "difficulty": "hard",
+      "naive_query": "醉酒过失致人重伤且有盗窃前科，应如何量刑？",
+      "subqueries": [
+        "过失伤害他人致人重伤 有期徒刑",
+        "危险驾驶罪 醉酒 驾驶机动车",
+        "累犯 从重处罚 过失犯罪"
+      ],
+      "gold_articles": ["第二百三十五条", "第一百三十三条之一", "第六十五条"]
+    }
+  ]
+}

chapter3/agentic-rag/evaluation/test_evaluate_empty_expected.py

@@ -0,0 +1,39 @@
+"""
+Test suite locking out ZeroDivisionError in RAGEvaluator.evaluate_response
+when test_case contains empty expected_keywords or expected_analysis lists.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
+
+from evaluate import RAGEvaluator
+
+
+def test_evaluate_response_empty_expected_keywords():
+    """
+    Ensure evaluate_response gracefully handles empty expected_keywords without raising ZeroDivisionError.
+    """
+    evaluator = RAGEvaluator.__new__(RAGEvaluator)
+    test_case = {
+        "id": "tc1",
+        "question": "Sample query?",
+        "expected_keywords": []
+    }
+    result = evaluator.evaluate_response("Sample answer", test_case)
+    assert result["metrics"]["keyword_recall"] == 1.0
+
+
+def test_evaluate_response_empty_expected_analysis():
+    """
+    Ensure evaluate_response gracefully handles empty expected_analysis without raising ZeroDivisionError.
+    """
+    evaluator = RAGEvaluator.__new__(RAGEvaluator)
+    test_case = {
+        "id": "tc2",
+        "question": "Sample query?",
+        "expected_analysis": []
+    }
+    result = evaluator.evaluate_response("Sample answer", test_case)
+    assert result["metrics"]["analysis_recall"] == 1.0