ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter7/tts-quality-eval/tests/conftest.py

@@ -0,0 +1,9 @@
+"""Test import bootstrap for the tts-quality-eval experiment."""
+
+from pathlib import Path
+import sys
+
+
+EXPERIMENT_ROOT = Path(__file__).resolve().parents[1]
+if str(EXPERIMENT_ROOT) not in sys.path:
+    sys.path.insert(0, str(EXPERIMENT_ROOT))

chapter7/tts-quality-eval/tests/test_judge_robustness.py

@@ -0,0 +1,254 @@
+"""
+Regression tests for judge-response robustness (实验 7-6 TTS 质量评估).
+
+Covers two failure classes on LLM/Gemini judge responses:
+  - judge_rubric: judge returns "score": null (or a bare null dimension) -> int(None) TypeError
+  - judge_gemini_audio: safety-blocked Gemini responses have no
+    candidates/content/parts -> KeyError/IndexError instead of a clear error
+
+Network is stubbed: the OpenAI-compatible judge client is replaced with a fake,
+and urllib.request.urlopen is monkeypatched for the Gemini REST call.
+"""
+import io
+import json
+
+import pytest
+
+import pipeline
+
+
+class _FakeMessage:
+    content = "{}"
+
+
+class _FakeChoice:
+    message = _FakeMessage()
+
+
+class _FakeResp:
+    choices = [_FakeChoice()]
+
+
+class _FakeCompletions:
+    @staticmethod
+    def create(**kwargs):
+        return _FakeResp()
+
+
+class _FakeChat:
+    completions = _FakeCompletions()
+
+
+class _FakeClient:
+    chat = _FakeChat()
+
+
+def _stub_judge(monkeypatch, payload: dict):
+    _FakeMessage.content = json.dumps(payload, ensure_ascii=False)
+    monkeypatch.setattr(
+        pipeline, "get_judge_client_and_model", lambda model=None: (_FakeClient(), "fake-judge"))
+
+
+def test_judge_rubric_tolerates_null_score(monkeypatch):
+    """'score': null in a dimension dict is scored 0, not int(None) TypeError."""
+    _stub_judge(monkeypatch, {
+        "准确性": {"score": None, "reason": "无法判断"},
+        "自然度": {"score": 4, "reason": "语速正常"},
+        "情感表达": {"score": 0},
+        "音色一致性": {"score": 0, "reason": "无法听到音频"},
+    })
+    rub = pipeline.judge_rubric("原文文本", "中性", "回译文本", 3.0, 0.05)
+    assert rub.scores["准确性"] == 0
+    assert rub.scores["自然度"] == 4
+    assert rub.scores["音色一致性"] == 0
+
+
+def test_judge_rubric_tolerates_null_dimension(monkeypatch):
+    """A bare null dimension (non-dict) is scored 0, not int(None) TypeError."""
+    _stub_judge(monkeypatch, {
+        "准确性": None,
+        "自然度": 4,
+        "情感表达": 0,
+        "音色一致性": 0,
+    })
+    rub = pipeline.judge_rubric("原文文本", "中性", "回译文本", 3.0, 0.05)
+    assert rub.scores["准确性"] == 0
+    assert rub.scores["自然度"] == 4
+
+
+class _FakeHTTPResp(io.BytesIO):
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        return False
+
+
+def _stub_gemini(monkeypatch, payload: dict):
+    monkeypatch.setenv("GEMINI_API_KEY", "fake-key-for-test")
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+    monkeypatch.delenv("MISTRAL_API_KEY", raising=False)
+    monkeypatch.setattr(pipeline, "_resolve_gemini_model", lambda key: "gemini-fake")
+    monkeypatch.setattr("urllib.request.urlopen",
+                        lambda req, timeout=None: _FakeHTTPResp(json.dumps(payload).encode()))
+
+
+@pytest.mark.parametrize("payload", [
+    {"promptFeedback": {"blockReason": "SAFETY"}},      # prompt 被拦截：无 candidates
+    {"candidates": []},                                  # 生成被拦截：空 candidates
+    {"candidates": [{"finishReason": "SAFETY", "index": 0}]},  # candidate 无 content
+])
+def test_judge_gemini_audio_blocked_raises_clear_error(monkeypatch, tmp_path, payload):
+    """Blocked/empty Gemini responses raise a clear RuntimeError, not KeyError/IndexError."""
+    _stub_gemini(monkeypatch, payload)
+    audio = tmp_path / "a.mp3"
+    audio.write_bytes(b"\xff\xfb" + b"\x00" * 256)
+    reference = tmp_path / "reference.mp3"
+    reference.write_bytes(b"\xff\xfb" + b"\x01" * 256)
+    with pytest.raises(RuntimeError, match="Gemini 未返回评审文本"):
+        pipeline.judge_gemini_audio("原文", "中性", str(audio), str(reference))
+
+
+def test_judge_gemini_audio_parses_valid_response(monkeypatch, tmp_path):
+    """A normal Gemini response still parses (defensive navigation keeps working)."""
+    inner = json.dumps({
+        "准确性": {"score": 4, "reason": "ok"},
+        "自然度": 4,
+        "情感表达": None,
+        "音色一致性": {"score": 5},
+    }, ensure_ascii=False)
+    _stub_gemini(monkeypatch, {
+        "candidates": [{"content": {"parts": [{"text": inner}]}}],
+    })
+    audio = tmp_path / "a.mp3"
+    audio.write_bytes(b"\xff\xfb" + b"\x00" * 256)
+    reference = tmp_path / "reference.mp3"
+    reference.write_bytes(b"\xff\xfb" + b"\x01" * 256)
+    rub = pipeline.judge_gemini_audio("原文", "中性", str(audio), str(reference))
+    assert rub.scores["准确性"] == 4
+    assert rub.scores["情感表达"] == 0   # null score -> 0
+    assert rub.scores["音色一致性"] == 5
+
+
+def test_judge_gemini_audio_falls_back_to_openrouter(monkeypatch, tmp_path):
+    """An unavailable direct key keeps both clips on a direct-audio fallback route."""
+    monkeypatch.setenv("GEMINI_API_KEY", "invalid-direct-key")
+    monkeypatch.setenv("OPENROUTER_API_KEY", "fake-openrouter-key")
+    monkeypatch.setattr(pipeline, "_resolve_gemini_model", lambda key: "gemini-fake")
+
+    def _http_error(req, timeout=None):
+        import urllib.error
+        raise urllib.error.HTTPError(req.full_url, 400, "bad key", {}, io.BytesIO(b"invalid"))
+
+    monkeypatch.setattr("urllib.request.urlopen", _http_error)
+    expected = pipeline.RubricResult(
+        scores={dim: 4 for dim in pipeline.RUBRIC_DIMENSIONS},
+        reasons={dim: "audible evidence" for dim in pipeline.RUBRIC_DIMENSIONS},
+        judge_model="openrouter/google/gemini-3.5-flash",
+        evidence_mode="direct-audio-with-reference",
+        provider_attempts=[],
+    )
+    monkeypatch.setattr(pipeline, "_judge_openrouter_audio", lambda *args, **kwargs: expected)
+    audio = tmp_path / "a.mp3"
+    audio.write_bytes(b"\xff\xfb" + b"\x00" * 256)
+    reference = tmp_path / "reference.mp3"
+    reference.write_bytes(b"\xff\xfb" + b"\x01" * 256)
+
+    rub = pipeline.judge_gemini_audio("原文", "中性", str(audio), str(reference))
+    assert rub is expected
+    assert rub.evidence_mode == "direct-audio-with-reference"
+
+
+def test_openrouter_failure_preserves_both_route_attempts(monkeypatch, tmp_path):
+    monkeypatch.setenv("GEMINI_API_KEY", "invalid-direct-key")
+    monkeypatch.setenv("OPENROUTER_API_KEY", "invalid-openrouter-key")
+    monkeypatch.delenv("MISTRAL_API_KEY", raising=False)
+    monkeypatch.setattr(pipeline, "_resolve_gemini_model", lambda key: "gemini-fake")
+
+    def _http_error(req, timeout=None):
+        import urllib.error
+        body = b"direct invalid" if "googleapis" in req.full_url else b"router invalid"
+        code = 400 if "googleapis" in req.full_url else 401
+        raise urllib.error.HTTPError(req.full_url, code, "unavailable", {}, io.BytesIO(body))
+
+    monkeypatch.setattr("urllib.request.urlopen", _http_error)
+    audio = tmp_path / "a.mp3"
+    audio.write_bytes(b"\xff\xfb" + b"\x00" * 256)
+    reference = tmp_path / "reference.mp3"
+    reference.write_bytes(b"\xff\xfb" + b"\x01" * 256)
+
+    with pytest.raises(pipeline.JudgeRouteError) as caught:
+        pipeline.judge_gemini_audio("原文", "中性", str(audio), str(reference))
+    assert [attempt["status"] for attempt in caught.value.provider_attempts] == [
+        "unavailable", "unavailable"
+    ]
+    assert [attempt["provider"] for attempt in caught.value.provider_attempts] == [
+        "Google Gemini API", "OpenRouter audio route"
+    ]
+
+
+def test_openrouter_failure_falls_back_to_exact_mistral_two_audio_route(monkeypatch, tmp_path):
+    monkeypatch.setenv("GEMINI_API_KEY", "invalid-direct-key")
+    monkeypatch.setenv("OPENROUTER_API_KEY", "invalid-openrouter-key")
+    monkeypatch.setenv("MISTRAL_API_KEY", "fake-mistral-key")
+    monkeypatch.setattr(pipeline, "_resolve_gemini_model", lambda key: "gemini-fake")
+    monkeypatch.setattr(pipeline.time, "sleep", lambda seconds: None)
+    observed_mistral_body = {}
+    mistral_calls = 0
+    inner = json.dumps({
+        dim: {"score": 4, "reason": "audible evidence"}
+        for dim in pipeline.RUBRIC_DIMENSIONS
+    }, ensure_ascii=False)
+
+    def _route(req, timeout=None):
+        nonlocal mistral_calls
+        import urllib.error
+        if "googleapis" in req.full_url:
+            raise urllib.error.HTTPError(
+                req.full_url, 400, "bad key", {}, io.BytesIO(b"direct invalid")
+            )
+        if "openrouter" in req.full_url:
+            raise urllib.error.HTTPError(
+                req.full_url, 401, "bad key", {}, io.BytesIO(b"router invalid")
+            )
+        mistral_calls += 1
+        if mistral_calls == 1:
+            raise urllib.error.HTTPError(
+                req.full_url, 500, "transient", {}, io.BytesIO(b"service unavailable")
+            )
+        observed_mistral_body.update(json.loads(req.data))
+        return _FakeHTTPResp(json.dumps({
+            "choices": [{"message": {"content": inner}}]
+        }).encode())
+
+    monkeypatch.setattr("urllib.request.urlopen", _route)
+    audio_bytes = b"\xff\xfb" + b"\x00" * 256
+    reference_bytes = b"\xff\xfb" + b"\x01" * 256
+    audio = tmp_path / "a.mp3"
+    audio.write_bytes(audio_bytes)
+    reference = tmp_path / "reference.mp3"
+    reference.write_bytes(reference_bytes)
+
+    rub = pipeline.judge_gemini_audio("原文", "中性", str(audio), str(reference))
+
+    content = observed_mistral_body["messages"][0]["content"]
+    assert [item["type"] for item in content] == [
+        "text", "text", "input_audio", "text", "input_audio"
+    ]
+    audio_chunks = [item for item in content if item["type"] == "input_audio"]
+    assert len(audio_chunks) == 2
+    assert audio_chunks[0]["input_audio"].startswith("data:audio/mpeg;base64,")
+    assert audio_chunks[1]["input_audio"].startswith("data:audio/mpeg;base64,")
+    assert rub.judge_model == "mistral/voxtral-small-latest"
+    assert rub.evidence_mode == "direct-audio-with-reference"
+    assert [attempt["provider"] for attempt in rub.provider_attempts] == [
+        "Google Gemini API", "OpenRouter audio route", "Mistral Voxtral API"
+    ]
+    assert [attempt["status"] for attempt in rub.provider_attempts] == [
+        "unavailable", "unavailable", "ok"
+    ]
+    assert rub.provider_attempts[-1]["attempts"] == 2
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

chapter7/tts-quality-eval/tests/test_resolve_gemini_model_null.py

@@ -0,0 +1,25 @@
+"""
+Test suite locking out TypeError in _resolve_gemini_model
+when API returns data with models: None or non-dict items.
+"""
+
+import json
+from unittest.mock import MagicMock
+
+from pipeline import _resolve_gemini_model, config
+
+
+def test_resolve_gemini_model_handles_null_models():
+    """
+    Ensure _resolve_gemini_model returns default model without TypeError when models key is None.
+    """
+    mock_resp = MagicMock()
+    mock_resp.read.return_value = json.dumps({"models": None}).encode("utf-8")
+    mock_resp.__enter__ = MagicMock(return_value=mock_resp)
+    mock_resp.__exit__ = MagicMock(return_value=None)
+
+    import urllib.request
+    urllib.request.urlopen = MagicMock(return_value=mock_resp)
+
+    res = _resolve_gemini_model("dummy_key")
+    assert res == config.GEMINI_MODEL_DEFAULT