ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter7/public-health-reporting-eval/tests/conftest.py

@@ -0,0 +1,9 @@
+"""Test import bootstrap for the public-health-reporting-eval experiment."""
+
+from pathlib import Path
+import sys
+
+
+EXPERIMENT_ROOT = Path(__file__).resolve().parents[1]
+if str(EXPERIMENT_ROOT) not in sys.path:
+    sys.path.insert(0, str(EXPERIMENT_ROOT))

chapter7/public-health-reporting-eval/tests/test_blank_csv_int.py

@@ -0,0 +1,76 @@
+"""Blank integer CSV cells must load as 0, not ValueError from int('')."""
+import csv
+from pathlib import Path
+
+from reporting_tools import ReportingEnvironment
+
+
+def _write_csv(path: Path, tests_value: str) -> None:
+    with path.open("w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(
+            handle,
+            fieldnames=[
+                "row_id",
+                "org_unit_id",
+                "period",
+                "parent_org_unit",
+                "tests",
+                "confirmed_cases",
+                "deaths",
+                "report_expected",
+                "report_submitted",
+                "stockout_days",
+            ],
+        )
+        writer.writeheader()
+        writer.writerow(
+            {
+                "row_id": "r1",
+                "org_unit_id": "ou1",
+                "period": "2024Q1",
+                "parent_org_unit": "p1",
+                "tests": tests_value,
+                "confirmed_cases": "5",
+                "deaths": "0",
+                "report_expected": "1",
+                "report_submitted": "1",
+                "stockout_days": "0",
+            }
+        )
+
+
+def test_blank_tests_field_loads_as_zero(tmp_path):
+    path = tmp_path / "blank.csv"
+    _write_csv(path, "")
+    env = ReportingEnvironment(path)
+    assert env.rows[0]["tests"] == 0
+    assert env.rows[0]["confirmed_cases"] == 5
+
+
+def test_whitespace_tests_field_loads_as_zero(tmp_path):
+    path = tmp_path / "space.csv"
+    _write_csv(path, "  ")
+    env = ReportingEnvironment(path)
+    assert env.rows[0]["tests"] == 0
+
+
+def test_numeric_tests_unchanged(tmp_path):
+    path = tmp_path / "ok.csv"
+    _write_csv(path, "12")
+    env = ReportingEnvironment(path)
+    assert env.rows[0]["tests"] == 12
+
+
+def test_reporting_environment_call_null_arguments(tmp_path):
+    """JSON null tool arguments should behave like an empty object."""
+    import pytest
+
+    path = tmp_path / "ok.csv"
+    _write_csv(path, "12")
+    env = ReportingEnvironment(path)
+    with pytest.raises(TypeError) as null_exc:
+        env.call("calculate_test_positivity", None)
+    with pytest.raises(TypeError) as empty_exc:
+        env.call("calculate_test_positivity", {})
+    assert "mapping" not in str(null_exc.value)
+    assert str(null_exc.value) == str(empty_exc.value)

chapter7/public-health-reporting-eval/tests/test_excel_float_int.py

@@ -0,0 +1,71 @@
+"""Excel-style whole floats in integer CSV cells must load (10.0 -> 10)."""
+import csv
+from pathlib import Path
+
+from reporting_tools import ReportingEnvironment
+
+
+def _write_csv(path: Path, tests_value: str) -> None:
+    with path.open("w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(
+            handle,
+            fieldnames=[
+                "row_id",
+                "org_unit_id",
+                "period",
+                "parent_org_unit",
+                "tests",
+                "confirmed_cases",
+                "deaths",
+                "report_expected",
+                "report_submitted",
+                "stockout_days",
+            ],
+        )
+        writer.writeheader()
+        writer.writerow(
+            {
+                "row_id": "r1",
+                "org_unit_id": "ou1",
+                "period": "2024Q1",
+                "parent_org_unit": "p1",
+                "tests": tests_value,
+                "confirmed_cases": "5",
+                "deaths": "0",
+                "report_expected": "1",
+                "report_submitted": "1",
+                "stockout_days": "0",
+            }
+        )
+
+
+def test_excel_float_tests_field_loads_as_int(tmp_path):
+    path = tmp_path / "excel.csv"
+    _write_csv(path, "10.0")
+    env = ReportingEnvironment(path)
+    assert env.rows[0]["tests"] == 10
+    assert env.rows[0]["confirmed_cases"] == 5
+
+
+def test_fractional_tests_still_rejected(tmp_path):
+    path = tmp_path / "frac.csv"
+    _write_csv(path, "3.5")
+    try:
+        ReportingEnvironment(path)
+        assert False, "expected ValueError"
+    except ValueError as e:
+        assert "non-integer" in str(e)
+
+
+def test_blank_still_zero(tmp_path):
+    path = tmp_path / "blank.csv"
+    _write_csv(path, "")
+    env = ReportingEnvironment(path)
+    assert env.rows[0]["tests"] == 0
+
+
+def test_plain_int_unchanged(tmp_path):
+    path = tmp_path / "ok.csv"
+    _write_csv(path, "12")
+    env = ReportingEnvironment(path)
+    assert env.rows[0]["tests"] == 12

chapter7/public-health-reporting-eval/tests/test_null_result.py

@@ -0,0 +1,71 @@
+"""score_prediction must tolerate result:null like missing/empty result."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from pathlib import Path
+
+from agent import DeterministicReportingAgent
+from evaluator import MAX_SCORE, expected_by_task, load_json, score_prediction
+from reporting_tools import ReportingEnvironment
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def _sample_expected():
+    expected = expected_by_task(ROOT / "expected_answers.json")
+    tasks = load_json(ROOT / "tasks.json")
+    environment = ReportingEnvironment(ROOT / "data" / "synthetic_reports.csv")
+    agent = DeterministicReportingAgent(environment, expected)
+    prediction = agent.run(tasks[0])
+    return prediction, expected[prediction["task_id"]]
+
+
+def test_null_result_scores_without_attribute_error():
+    prediction, expected = _sample_expected()
+    prediction = deepcopy(prediction)
+    prediction["result"] = None
+    result = score_prediction(prediction, expected)
+    assert result["details"]["answer"] == 0
+    assert result["details"]["evidence"] == 0
+    assert result["score"] == (
+        result["details"]["tool_selection"]
+        + result["details"]["arguments"]
+        + result["details"]["grounding_and_safety"]
+    )
+
+
+def test_missing_result_matches_null_result_score():
+    prediction, expected = _sample_expected()
+    null_pred = deepcopy(prediction)
+    null_pred["result"] = None
+    missing_pred = deepcopy(prediction)
+    del missing_pred["result"]
+    assert score_prediction(null_pred, expected) == score_prediction(missing_pred, expected)
+
+
+def test_valid_result_still_full_score():
+    prediction, expected = _sample_expected()
+    result = score_prediction(prediction, expected)
+    assert result["score"] == MAX_SCORE
+
+
+def test_unhashable_evidence_in_result():
+    prediction, expected = _sample_expected()
+    prediction = deepcopy(prediction)
+    prediction["result"]["evidence"] = [{"url": "http://example.com"}]
+    result = score_prediction(prediction, expected)
+    assert result["details"]["evidence"] == 0
+
+
+def test_unhashable_evidence_remains_order_independent():
+    prediction, expected = _sample_expected()
+    prediction = deepcopy(prediction)
+    expected = deepcopy(expected)
+    expected["result"]["evidence"] = [{"url": "a"}, {"url": "b"}]
+    prediction["result"]["evidence"] = [{"url": "b"}, {"url": "a"}]
+
+    result = score_prediction(prediction, expected)
+
+    assert result["details"]["evidence"] == 1

chapter7/public-health-reporting-eval/tests/test_offline.py

@@ -0,0 +1,68 @@
+"""Offline regression tests; no model, API key or network access required."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from pathlib import Path
+
+from agent import DeterministicReportingAgent
+from evaluator import MAX_SCORE, evaluate, expected_by_task, load_json, score_prediction
+from reporting_tools import ReportingEnvironment
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def reference_predictions():
+    expected = expected_by_task(ROOT / "expected_answers.json")
+    tasks = load_json(ROOT / "tasks.json")
+    environment = ReportingEnvironment(ROOT / "data" / "synthetic_reports.csv")
+    agent = DeterministicReportingAgent(environment, expected)
+    return [agent.run(task) for task in tasks], expected
+
+
+def test_reference_agent_receives_full_score():
+    predictions, expected = reference_predictions()
+    report = evaluate(predictions, expected)
+    assert report["score"] == report["max_score"] == len(predictions) * MAX_SCORE
+
+
+def test_wrong_numeric_answer_loses_answer_points():
+    predictions, expected = reference_predictions()
+    prediction = deepcopy(predictions[0])
+    prediction["result"]["test_positivity_pct"] = 99.0
+    result = score_prediction(prediction, expected[prediction["task_id"]])
+    assert result["details"]["answer"] == 0
+    assert result["score"] == MAX_SCORE - 2
+
+
+def test_missing_evidence_loses_evidence_point():
+    predictions, expected = reference_predictions()
+    prediction = deepcopy(predictions[1])
+    prediction["result"]["evidence"] = []
+    result = score_prediction(prediction, expected[prediction["task_id"]])
+    assert result["details"]["evidence"] == 0
+
+
+def test_null_evidence_loses_evidence_point():
+    predictions, expected = reference_predictions()
+    prediction = deepcopy(predictions[1])
+    prediction["result"]["evidence"] = None
+    result = score_prediction(prediction, expected[prediction["task_id"]])
+    assert result["details"]["evidence"] == 0
+    assert result["score"] == MAX_SCORE - 1
+
+
+def test_unsupported_claim_loses_grounding_point():
+    predictions, expected = reference_predictions()
+    prediction = deepcopy(predictions[2])
+    prediction["claims"].append("This trend proves an outbreak will occur.")
+    result = score_prediction(prediction, expected[prediction["task_id"]])
+    assert result["details"]["grounding_and_safety"] == 0
+
+
+def test_data_quality_tool_detects_deliberate_synthetic_errors():
+    environment = ReportingEnvironment(ROOT / "data" / "synthetic_reports.csv")
+    result = environment.find_data_quality_issues("Demo District", "2025-02")
+    assert result["issue_count"] == 2
+    assert result["evidence"] == ["R005"]