ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

tests/test_ch8_evaluate_multilingual.py

@@ -0,0 +1,295 @@
+"""Unit tests for chapter8/MultilingualReasoning/evaluate_multilingual.py."""
+
+from pathlib import Path
+import sys
+import warnings
+
+# Ensure chapter8/MultilingualReasoning is in sys.path
+ch8_dir = Path(__file__).resolve().parent.parent / "chapter8" / "MultilingualReasoning"
+if str(ch8_dir) not in sys.path:
+    sys.path.insert(0, str(ch8_dir))
+
+from evaluate_multilingual import (
+    MultilingualReasoningEvaluator,
+    normalize_language,
+    run_evaluation,
+)
+
+
+def test_normalize_language():
+    assert normalize_language("en") == "English"
+    assert normalize_language("SPANISH") == "Spanish"
+    assert normalize_language("fr") == "French"
+    assert normalize_language("zh") == "Chinese"
+    assert normalize_language("ja") == "Japanese"
+    assert normalize_language("German") == "German"
+
+
+def test_cot_fidelity_scoring():
+    evaluator = MultilingualReasoningEvaluator()
+
+    # Chinese CoT fidelity
+    zh_cot = "首先计算第一步：因为 2 + 2 = 4，所以结论是 4。"
+    assert evaluator.evaluate_cot_fidelity(zh_cot, "Chinese") > 0.8
+
+    # Japanese CoT fidelity (contains Hiragana and CJK)
+    ja_cot = "ステップ1：2 + 2 = 4 なので、答えは 4 です。"
+    assert evaluator.evaluate_cot_fidelity(ja_cot, "Japanese") > 0.8
+
+    # Spanish CoT fidelity
+    es_cot = "Paso 1: Porque 2 + 2 es igual a 4, entonces la respuesta es 4."
+    assert evaluator.evaluate_cot_fidelity(es_cot, "Spanish") > 0.5
+
+    # French CoT fidelity
+    fr_cot = "Étape 1: Parce que 2 + 2 est égal à 4, donc la réponse est 4."
+    assert evaluator.evaluate_cot_fidelity(fr_cot, "French") > 0.5
+
+    # English CoT fidelity
+    en_cot = "Step 1: Because 2 + 2 equals 4, therefore the answer is 4."
+    assert evaluator.evaluate_cot_fidelity(en_cot, "English") > 0.7
+
+    # Cross-lingual leakage (Chinese text evaluated as English fidelity)
+    assert evaluator.evaluate_cot_fidelity(zh_cot, "English") == 0.0
+
+
+def test_evaluate_accuracy():
+    evaluator = MultilingualReasoningEvaluator()
+
+    assert evaluator.evaluate_accuracy("42", "42") == 1.0
+    assert evaluator.evaluate_accuracy("42.0", "42") == 1.0
+    assert evaluator.evaluate_accuracy("The answer is 42.", "42") == 1.0
+    assert evaluator.evaluate_accuracy("Paris", "paris!") == 1.0
+    assert evaluator.evaluate_accuracy("Wrong", "42") == 0.0
+    assert evaluator.evaluate_accuracy("1042", "42") == 0.0
+    assert evaluator.evaluate_accuracy("0", 0) == 1.0
+
+def test_evaluator_sample_formats():
+    evaluator = MultilingualReasoningEvaluator()
+
+    # Mock model returning string with <think> tag
+    def string_model(prompt, language="English"):
+        return "<think>Step 1: Reasoning here.</think> 42"
+
+    sample = {
+        "language": "en",
+        "prompt": "What is 40 + 2?",
+        "reference_answer": "42",
+    }
+    res = evaluator.evaluate_sample(string_model, sample)
+    assert res["language"] == "English"
+    assert res["accuracy"] == 1.0
+    assert res["reasoning"] == "Step 1: Reasoning here."
+    assert res["predicted_answer"] == "42"
+
+    # Mock model returning dict
+    def dict_model(prompt, language="Spanish"):
+        return {
+            "reasoning": "Paso 1: Razonamiento en español.",
+            "answer": "42",
+            "token_usage": {"prompt_tokens": 10, "completion_tokens": 20, "reasoning_tokens": 15, "total_tokens": 30},
+        }
+
+    sample_es = {
+        "target_language": "Spanish",
+        "question": "¿Cuánto es 40 + 2?",
+        "ground_truth": "42",
+    }
+    res_es = evaluator.evaluate_sample(dict_model, sample_es)
+    assert res_es["language"] == "Spanish"
+    assert res_es["accuracy"] == 1.0
+    assert res_es["token_usage"]["total_tokens"] == 30
+
+    # Test non-falsy zero answer
+    sample_zero = {
+        "language": "en",
+        "prompt": "What is 2 - 2?",
+        "reference_answer": 0,
+    }
+    res_zero = evaluator.evaluate_sample(lambda p: "0", sample_zero)
+    assert res_zero["reference_answer"] == "0"
+    assert res_zero["accuracy"] == 1.0
+
+
+def test_run_evaluation_end_to_end():
+    dataset = [
+        {"language": "en", "prompt": "What is 2+2?", "reference_answer": "4"},
+        {"language": "es", "prompt": "¿Cuánto es 2+2?", "reference_answer": "4"},
+        {"language": "fr", "prompt": "Combien font 2+2?", "reference_answer": "4"},
+        {"language": "zh", "prompt": "2+2等于多少？", "reference_answer": "4"},
+        {"language": "ja", "prompt": "2+2はいくらですか？", "reference_answer": "4"},
+    ]
+
+    def mock_multilingual_model(prompt, language="English"):
+        responses = {
+            "English": "<think>Step 1: Add numbers.</think> 4",
+            "Spanish": "<think>Paso 1: Sumar números, entonces es 4.</think> 4",
+            "French": "<think>Étape 1: Additionner donc c'est 4.</think> 4",
+            "Chinese": "<think>第一步：因为 2+2=4，所以是 4。</think> 4",
+            "Japanese": "<think>ステップ1：2+2=4 なので 4 です。</think> 4",
+        }
+        return responses.get(language, "<think>Step 1</think> 4")
+
+    report = run_evaluation(mock_multilingual_model, dataset)
+
+    assert report["num_samples"] == 5
+    assert report["overall_accuracy"] == 1.0
+    assert report["overall_cot_fidelity"] > 0.6
+    assert report["overall_transfer_efficiency"] == 1.0
+    assert "English" in report["by_language"]
+    assert "Spanish" in report["by_language"]
+    assert "French" in report["by_language"]
+    assert "Chinese" in report["by_language"]
+    assert "Japanese" in report["by_language"]
+    assert report["total_token_usage"]["total_tokens"] > 0
+
+
+def test_run_evaluation_empty_dataset():
+    report = run_evaluation(lambda p: "42", [])
+    assert report["num_samples"] == 0
+    assert report["overall_accuracy"] == 0.0
+    assert report["by_language"] == {}
+
+
+def test_object_model_and_method_invocations():
+    evaluator = MultilingualReasoningEvaluator()
+
+    class CustomOutput:
+        def __init__(self):
+            self.reasoning = "Step 1: Compute."
+            self.answer = "42"
+            self.token_usage = {
+                "prompt_tokens": 10,
+                "completion_tokens": 20,
+                "reasoning_tokens": 15,
+                "total_tokens": 30,
+            }
+
+    class GenerateModel:
+        def generate(self, prompt, language="English"):
+            return CustomOutput()
+
+    class PredictModel:
+        def predict(self, prompt):
+            return "Reasoning: Simple math\nAnswer: 42"
+    sample = {"language": "en", "prompt": "40+2?", "reference_answer": "42"}
+    res_gen = evaluator.evaluate_sample(GenerateModel(), sample)
+    assert res_gen["accuracy"] == 1.0
+    assert res_gen["token_usage"]["total_tokens"] == 30
+
+    res_pred = evaluator.evaluate_sample(PredictModel(), sample)
+    assert res_pred["accuracy"] == 1.0
+
+
+def test_transfer_efficiency_zero_reference():
+    evaluator = MultilingualReasoningEvaluator()
+    metrics = {
+        "English": {"accuracy": 0.0},
+        "Spanish": {"accuracy": 0.0},
+    }
+    eff = evaluator.compute_transfer_efficiency(metrics)
+    assert eff["English"] == 0.0
+    assert eff["Spanish"] == 0.0
+
+
+def test_model_exception_and_builtin_callable():
+    evaluator = MultilingualReasoningEvaluator()
+
+    def failing_model(prompt):
+        raise RuntimeError("Model inference failed")
+
+    dataset = [{"language": "en", "prompt": "test", "reference_answer": "42"}]
+    report = evaluator.evaluate(failing_model, dataset)
+    assert report["num_samples"] == 1
+    assert report["overall_accuracy"] == 0.0
+def test_token_usage_object_attributes():
+    evaluator = MultilingualReasoningEvaluator()
+
+    class TokenUsageObj:
+        def __init__(self, input_tokens=12, output_tokens=24, total_tokens=36):
+            self.input_tokens = input_tokens
+            self.output_tokens = output_tokens
+            self.total_tokens = total_tokens
+
+    class ObjectOutputModel:
+        def __init__(self):
+            self.token_usage = TokenUsageObj()
+
+        def generate(self, prompt):
+            return {"answer": "42", "token_usage": TokenUsageObj(input_tokens=15, output_tokens=30, total_tokens=45)}
+
+    sample = {"language": "en", "prompt": "What is 40+2?", "reference_answer": "42"}
+    res = evaluator.evaluate_sample(ObjectOutputModel(), sample)
+    assert res["token_usage"]["prompt_tokens"] == 15
+    assert res["token_usage"]["completion_tokens"] == 30
+    assert res["token_usage"]["total_tokens"] == 45
+
+    # Direct test on compute_token_usage with token usage object
+    tu_obj = TokenUsageObj(input_tokens=100, output_tokens=200, total_tokens=300)
+    res_direct = evaluator.compute_token_usage("prompt", "reasoning", "answer", model_output=tu_obj)
+    assert res_direct["prompt_tokens"] == 100
+    assert res_direct["completion_tokens"] == 200
+    assert res_direct["total_tokens"] == 300
+
+
+def test_word_boundary_reference_matching():
+    evaluator = MultilingualReasoningEvaluator()
+    # Word boundary matching should succeed for full word substring
+    assert evaluator.evaluate_accuracy("The answer is Paris.", "Paris") == 1.0
+    assert evaluator.evaluate_accuracy("The answer is A", "A") == 1.0
+    # Word boundary matching should fail for partial word matching
+    assert evaluator.evaluate_accuracy("1042", "42") == 0.0
+    assert evaluator.evaluate_accuracy("no", "not paris") == 0.0
+    assert evaluator.evaluate_accuracy("apple", "a") == 0.0
+
+def test_invoke_model_inspect_signature():
+    evaluator = MultilingualReasoningEvaluator()
+
+    # Function accepting language
+    def model_with_lang(prompt, language="English"):
+        return f"Response for {language}: {prompt}"
+
+    # Function not accepting language
+    def model_without_lang(prompt):
+        return f"Response: {prompt}"
+
+    sample = {"language": "Spanish", "prompt": "Hola", "reference_answer": "Hola"}
+    res_lang = evaluator.evaluate_sample(model_with_lang, sample)
+    assert "Spanish" in res_lang["predicted_answer"]
+
+    res_nolang = evaluator.evaluate_sample(model_without_lang, sample)
+    assert "Response: Hola" == res_nolang["predicted_answer"]
+def test_zero_answer_handling():
+    evaluator = MultilingualReasoningEvaluator()
+    sample = {"language": "en", "prompt": "1-1?", "reference_answer": 0}
+    model = lambda p: {"answer": 0, "reasoning": "1 minus 1 equals 0"}
+    res = evaluator.evaluate_sample(model, sample)
+    assert res["predicted_answer"] == "0"
+    assert res["reference_answer"] == "0"
+    assert res["accuracy"] == 1.0
+def test_chinese_cot_fidelity_japanese_kana_penalty():
+    evaluator = MultilingualReasoningEvaluator()
+    # Chinese CoT containing Japanese kana should be penalized (capped at 0.7)
+    cot_with_kana = "第一歩：計算結果、二足す二は四、答案は四。だ"
+    score = evaluator.evaluate_cot_fidelity(cot_with_kana, "Chinese")
+    assert score == 0.7
+
+
+def test_transfer_efficiency_none_accuracy():
+    evaluator = MultilingualReasoningEvaluator()
+    metrics = {
+        "English": {"accuracy": None},
+        "Spanish": {"accuracy": 0.5},
+    }
+    eff = evaluator.compute_transfer_efficiency(metrics)
+    assert eff["English"] == 0.0
+    assert eff["Spanish"] == 1.0
+
+
+def test_partial_token_usage_reasoning_estimation():
+    evaluator = MultilingualReasoningEvaluator()
+    tu = {"prompt_tokens": 10, "completion_tokens": 50, "total_tokens": 60}
+    res = evaluator.compute_token_usage("prompt", "detailed reasoning step by step", "answer", model_output=tu)
+    assert res["prompt_tokens"] == 10
+    assert res["completion_tokens"] == 50
+    assert res["reasoning_tokens"] > 0
+    assert res["total_tokens"] == 60