ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter5/agent-creator/.gitignore

@@ -0,0 +1,5 @@
+.env
+__pycache__/
+.pytest_cache/
+runs/
+*.pyc

chapter5/agent-creator/README.md

@@ -0,0 +1,62 @@
+# Experiment 5-13: An Agent That Creates Agents
+
+This is the runnable companion for Chapter 5, Experiment 5-13. It implements the
+book's complete comparison rather than merely pointing at `coding-agent` as a
+possible starting point.
+
+The experiment asks the same real model to create two specialized Agents:
+
+1. **From scratch**: generate the Agent loop, tool protocol, domain tools, CLI,
+   and tests with no reference implementation.
+2. **Template adaptation**: copy the proven `reference_agent`, preserve its
+   standard message/tool loop, and generate only the domain-specific prompt,
+   tool schemas, implementations, documentation, and tests.
+
+Both outputs pass the same gates:
+
+- required-file and secret scan;
+- Python AST/compile validation;
+- standard `assistant.tool_calls → role=tool` protocol audit;
+- bounded-loop audit;
+- generated pytest suite;
+- a real API run of the generated Agent on its own sample task.
+
+The resulting `comparison.json` records generation time and token use, every
+validation gate, the live Agent trace, and the winning strategy. There is no
+mock fallback in the default experiment: missing credentials or a failed live
+Agent run fails the command.
+
+## Run
+
+```bash
+cd chapter5/agent-creator
+pip install -r requirements.txt
+cp env.example .env
+python demo.py --output runs/release-agent
+```
+
+Use a custom target:
+
+```bash
+python demo.py \
+  --requirements "Create an incident triage Agent that queries service health and drafts an evidence-backed escalation" \
+  --output runs/incident-triage
+```
+
+`--no-live` exists only for deterministic CI/unit testing. It is not considered
+a completed experiment run.
+
+## Files
+
+- `creator.py`: real-model creator and the two controlled comparison arms.
+- `reference_agent/`: the known-good Agent that template mode copies.
+- `validator.py`: common structural, test, and live-runtime gates.
+- `demo.py`: one-command end-to-end comparison.
+- `test_creator.py`: creator safety and orchestration tests.
+
+## Security boundary
+
+Generated paths are allowlisted, credentials are never placed in prompts or
+generated files, and live execution occurs only after structural and test gates.
+Generated domain tools still execute local code, so review them before using the
+output outside an isolated experiment directory.

chapter5/agent-creator/demo.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+try:
+    from dotenv import load_dotenv
+    load_dotenv()
+except ImportError:
+    pass
+
+from creator import DEFAULT_PROTOCOL, load_protocol, run_experiment
+
+
+DEFAULT_REQUIREMENTS = load_protocol()[0]["requirements"]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Experiment 5-13: compare an Agent created from scratch with one adapted from a proven Agent"
+    )
+    parser.add_argument("--requirements", default=DEFAULT_REQUIREMENTS)
+    parser.add_argument("--output", type=Path, default=Path("runs/latest"))
+    parser.add_argument("--protocol", type=Path, default=DEFAULT_PROTOCOL)
+    parser.add_argument(
+        "--live-task",
+        default=None,
+        help="Development-only single task override; it cannot complete the frozen book experiment",
+    )
+    parser.add_argument("--no-live", action="store_true", help="Skip real API execution of generated Agents")
+    parser.add_argument(
+        "--resume",
+        action="store_true",
+        help="Reuse already generated arms in --output and repair/revalidate them",
+    )
+    args = parser.parse_args()
+    result = run_experiment(
+        args.requirements,
+        args.output,
+        live=not args.no_live,
+        live_task=args.live_task,
+        resume=args.resume,
+        protocol_path=args.protocol,
+    )
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    if not result["official_complete"]:
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

chapter5/agent-creator/env.example

@@ -0,0 +1,24 @@
+# Provider selection: auto prefers Moonshot/Kimi, then Ark, direct OpenAI,
+# then OpenRouter.  The creator and both generated Agents use the same endpoint.
+AGENT_CREATOR_PROVIDER=auto
+AGENT_CREATOR_MODEL=
+
+# Moonshot/Kimi (recommended for the book's current default experiment).
+MOONSHOT_API_KEY=
+KIMI_API_KEY=
+KIMI_MODEL=kimi-k3
+MOONSHOT_BASE_URL=https://api.moonshot.cn/v1
+
+# Volcengine Ark (ARK_MODEL must be a callable endpoint/model id).
+ARK_API_KEY=
+ARK_MODEL=
+ARK_BASE_URL=https://ark.cn-beijing.volces.com/api/v3
+
+# Direct OpenAI-compatible endpoint.
+OPENAI_API_KEY=
+OPENAI_BASE_URL=
+OPENAI_MODEL=gpt-5.6-luna
+
+# Universal fallback when a direct OpenAI key/model is unavailable.
+OPENROUTER_API_KEY=
+OPENROUTER_MODEL=openai/gpt-5.6-luna

chapter5/agent-creator/experiment_protocol.json

@@ -0,0 +1,133 @@
+{
+  "schema_version": "2.0",
+  "experiment": "5-13",
+  "frozen_at_utc": "2026-07-30T00:00:00Z",
+  "manuscript_source": "book/chapter5.md#experiment-5-13",
+  "requirements": "Create a release-readiness Agent. It must inspect structured deployment facts, identify failed quality gates, refuse release when any required gate fails, and produce an evidence-backed remediation checklist.",
+  "backend_requirement": {
+    "provider": "moonshot",
+    "model": "kimi-k3",
+    "api_style": "OpenAI-compatible chat.completions with tools/tool_calls",
+    "documentation_url": "https://platform.kimi.com/docs/guide/start-using-kimi-api",
+    "pricing": {
+      "as_of": "2026-07-29",
+      "currency": "CNY",
+      "uncached_input_per_million": 20.0,
+      "cached_input_per_million": 2.0,
+      "output_per_million": 100.0,
+      "source_url": "https://platform.kimi.com/docs/pricing/chat-k3.md",
+      "legacy_or_missing_cache_split_policy": "Treat all prompt tokens without an observed cached-token split as uncached."
+    }
+  },
+  "comparison_design": {
+    "strategies": [
+      "template",
+      "scratch"
+    ],
+    "controlled_variables": [
+      "creator provider and model",
+      "generated-Agent provider and model",
+      "requirements",
+      "live cases and histories",
+      "deterministic validation code",
+      "timeouts and maximum repair attempts"
+    ],
+    "quality_metric": "Sum of preregistered deterministic case checks. Quality non-inferiority means template score >= scratch score; strict advantage means template score > scratch score.",
+    "efficiency_metric": "Creation only. Template must use fewer total creator tokens (prompt + completion) and less creator wall time than scratch. Live task cost is reported separately and is not used to choose the creation winner.",
+    "joint_book_claim": "Supported only when template has a strict quality advantage and an efficiency advantage. A quality tie is reported as non-inferior, never as a strict quality advantage.",
+    "no_post_hoc_rule": "This file and its SHA-256 are saved with the campaign. Changing any criterion requires a new protocol version and a new campaign."
+  },
+  "completion_gates": [
+    "protocol hash recorded",
+    "required current provider, model, and API style used",
+    "both arms generated by the same real model",
+    "both arms pass required-file, secret, compile, and generated-test gates",
+    "both arms use standard assistant.tool_calls followed by matching role=tool messages",
+    "both arms run every common real basic task",
+    "both arms preserve supplied multi-turn history and use it in the final answer",
+    "credential-free raw creator and live evidence saved",
+    "provider usage saved with complete native-currency cost accounting",
+    "quality and efficiency conclusions computed from this frozen protocol"
+  ],
+  "live_cases": [
+    {
+      "id": "refuse_failed_and_skipped",
+      "kind": "basic_task",
+      "history": [],
+      "task": "Evaluate this release candidate and produce the final evidence-backed decision and remediation checklist without asking for more information: {\"deployment\":\"payment-service:v2.4.1\",\"environment\":\"production\",\"gates\":[{\"id\":\"unit_tests\",\"required\":true,\"outcome\":\"passed\",\"evidence\":\"1842/1842 tests passed\"},{\"id\":\"integration_tests\",\"required\":true,\"outcome\":\"failed\",\"evidence\":\"checkout_webhook test failed\"},{\"id\":\"security_scan\",\"required\":true,\"outcome\":\"passed\",\"evidence\":\"0 critical or high findings\"},{\"id\":\"load_test\",\"required\":true,\"outcome\":\"skipped\",\"evidence\":\"no report uploaded\"},{\"id\":\"code_review\",\"required\":true,\"outcome\":\"passed\",\"evidence\":\"two approvals\"}]}. A required gate passes only when outcome is exactly passed.",
+      "expected": {
+        "decision": "REFUSED",
+        "failed_ids": [
+          "integration_tests",
+          "load_test"
+        ],
+        "evidence": [
+          "checkout_webhook test failed",
+          "no report uploaded"
+        ],
+        "answer_substrings": [
+          "REFUSED",
+          "integration_tests",
+          "load_test"
+        ],
+        "forbidden_answer_substrings": [
+          "APPROVED"
+        ],
+        "context_markers": []
+      }
+    },
+    {
+      "id": "approve_required_optional_failure",
+      "kind": "basic_task",
+      "history": [],
+      "task": "Evaluate this release candidate and give the final evidence-backed decision: {\"deployment\":\"catalog-service:v1.8.0\",\"environment\":\"staging\",\"gates\":[{\"id\":\"unit_tests\",\"required\":true,\"outcome\":\"passed\",\"evidence\":\"912/912 tests passed\"},{\"id\":\"security_scan\",\"required\":true,\"outcome\":\"passed\",\"evidence\":\"0 high findings\"},{\"id\":\"optional_benchmark\",\"required\":false,\"outcome\":\"failed\",\"evidence\":\"optional latency target missed\"}]}. Optional failures do not block release.",
+      "expected": {
+        "decision": "APPROVED",
+        "failed_ids": [],
+        "evidence": [],
+        "answer_substrings": [
+          "APPROVED"
+        ],
+        "forbidden_answer_substrings": [
+          "REFUSED"
+        ],
+        "context_markers": []
+      }
+    },
+    {
+      "id": "multiturn_state_and_refusal",
+      "kind": "multi_turn_state",
+      "history": [
+        {
+          "role": "user",
+          "content": "For the next release decision, remember that the accountable release owner is Mei-Lin and the change ticket is CR-4821."
+        },
+        {
+          "role": "assistant",
+          "content": "Understood. I will retain release owner Mei-Lin and change ticket CR-4821 for the next decision."
+        }
+      ],
+      "task": "Using the prior conversation state, name the release owner and change ticket, then evaluate: {\"deployment\":\"identity-service:v3.0.0\",\"environment\":\"production\",\"gates\":[{\"id\":\"unit_tests\",\"required\":true,\"outcome\":\"passed\",\"evidence\":\"2201/2201 tests passed\"},{\"id\":\"rollback_drill\",\"required\":true,\"outcome\":\"failed\",\"evidence\":\"rollback exceeded the 10-minute objective\"}]}. Give a final evidence-backed decision and remediation.",
+      "expected": {
+        "decision": "REFUSED",
+        "failed_ids": [
+          "rollback_drill"
+        ],
+        "evidence": [
+          "rollback exceeded the 10-minute objective"
+        ],
+        "answer_substrings": [
+          "REFUSED",
+          "rollback_drill"
+        ],
+        "forbidden_answer_substrings": [
+          "APPROVED"
+        ],
+        "context_markers": [
+          "Mei-Lin",
+          "CR-4821"
+        ]
+      }
+    }
+  ]
+}

chapter5/agent-creator/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+norecursedirs = runs
+testpaths = . reference_agent/tests
+python_files = test_*.py

chapter5/agent-creator/reference_agent/agent.py

@@ -0,0 +1,139 @@
+"""A small production-shaped OpenAI-compatible Agent loop.
+
+The creator preserves this loop in template mode and only specializes the
+system prompt, tool schemas, and domain tool implementation.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from openai import OpenAI
+
+from domain_tools import execute_tool
+
+
+ROOT = Path(__file__).resolve().parent
+
+
+def _load_json(path: Path) -> Any:
+    with path.open(encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+class GeneratedAgent:
+    def __init__(self, *, model: str | None = None, client: Any | None = None):
+        self.model = model or os.getenv("OPENAI_MODEL") or os.getenv(
+            "OPENROUTER_MODEL", "openai/gpt-5.6-luna"
+        )
+        use_router = bool(os.getenv("OPENROUTER_API_KEY")) and (
+            "/" in self.model
+            or os.getenv("AGENT_PROVIDER", "auto").casefold() in {"auto", "openrouter"}
+        )
+        api_key = os.getenv("OPENROUTER_API_KEY") if use_router else os.getenv("OPENAI_API_KEY")
+        base_url = "https://openrouter.ai/api/v1" if use_router else os.getenv("OPENAI_BASE_URL")
+        if client is None and not api_key:
+            raise RuntimeError("Set OPENAI_API_KEY or OPENROUTER_API_KEY")
+        self.client = client or OpenAI(api_key=api_key, base_url=base_url)
+        self.system_prompt = (ROOT / "system_prompt.md").read_text(encoding="utf-8")
+        self.tools = _load_json(ROOT / "tools.json")["tools"]
+
+    @staticmethod
+    def _assistant_message(message: Any) -> dict[str, Any]:
+        result: dict[str, Any] = {"role": "assistant", "content": message.content or ""}
+        if message.tool_calls:
+            result["tool_calls"] = [
+                {
+                    "id": call.id,
+                    "type": "function",
+                    "function": {
+                        "name": call.function.name,
+                        "arguments": call.function.arguments,
+                    },
+                }
+                for call in message.tool_calls
+            ]
+        return result
+
+    def run(
+        self,
+        task: str,
+        *,
+        history: list[dict[str, Any]] | None = None,
+        max_iterations: int = 12,
+    ) -> dict[str, Any]:
+        messages: list[dict[str, Any]] = [
+            {"role": "system", "content": self.system_prompt},
+            *(history or []),
+            {"role": "user", "content": task},
+        ]
+        trace: list[dict[str, Any]] = []
+        usage_totals = {
+            "prompt_tokens": 0,
+            "cached_prompt_tokens": 0,
+            "completion_tokens": 0,
+            "requests": 0,
+        }
+        for iteration in range(1, max_iterations + 1):
+            kwargs = dict(
+                model=self.model,
+                messages=messages,
+                tools=self.tools,
+                tool_choice="auto",
+            )
+            if any(tag in self.model.casefold() for tag in ("kimi-", "gpt-5")):
+                kwargs["temperature"] = 1
+            else:
+                kwargs["temperature"] = 0
+            response = self.client.chat.completions.create(**kwargs)
+            message = response.choices[0].message
+            messages.append(self._assistant_message(message))
+            usage = getattr(response, "usage", None)
+            prompt_details = getattr(usage, "prompt_tokens_details", None)
+            usage_totals["prompt_tokens"] += getattr(usage, "prompt_tokens", 0) or 0
+            usage_totals["cached_prompt_tokens"] += (
+                getattr(prompt_details, "cached_tokens", 0) or 0
+            )
+            usage_totals["completion_tokens"] += (
+                getattr(usage, "completion_tokens", 0) or 0
+            )
+            usage_totals["requests"] += 1
+            trace.append({
+                "iteration": iteration,
+                "content": message.content or "",
+                "tool_calls": len(message.tool_calls or []),
+                "prompt_tokens": getattr(usage, "prompt_tokens", None),
+                "completion_tokens": getattr(usage, "completion_tokens", None),
+            })
+            if not message.tool_calls:
+                return {
+                    "ok": True,
+                    "answer": message.content or "",
+                    "iterations": iteration,
+                    "trace": trace,
+                    "messages": messages,
+                    "usage": usage_totals,
+                }
+            for call in message.tool_calls:
+                try:
+                    arguments = json.loads(call.function.arguments or "{}")
+                    result = execute_tool(call.function.name, arguments)
+                except Exception as exc:  # tool failures must return to the model
+                    result = {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
+                messages.append({
+                    "role": "tool",
+                    "tool_call_id": call.id,
+                    "content": json.dumps(result, ensure_ascii=False),
+                })
+        return {
+            "ok": False,
+            "answer": "",
+            "iterations": max_iterations,
+            "trace": trace,
+            "messages": messages,
+            "usage": usage_totals,
+            "error": "maximum iterations reached",
+        }

chapter5/agent-creator/reference_agent/domain_spec.json

@@ -0,0 +1,24 @@
+{
+  "schema_version": "1.0",
+  "name": "Reference Policy Agent",
+  "role": "Evaluate structured policy records using only supplied evidence.",
+  "requirements": "Demonstrate the uncustomized policy-record template.",
+  "sample_task": "Evaluate the supplied checks.",
+  "tool_name": "evaluate_required_records",
+  "tool_description": "Evaluate every user-supplied record against its required passing state.",
+  "record_noun": "policy record",
+  "records_argument": "records",
+  "identifier_field": "id",
+  "required_field": "required",
+  "status_field": "status",
+  "evidence_field": "evidence",
+  "passing_values": [
+    "passed"
+  ],
+  "approved_label": "APPROVED",
+  "rejected_label": "REFUSED",
+  "remediation_by_status": {
+    "failed": "Correct the failed requirement and rerun it."
+  },
+  "default_remediation": "Resolve the non-passing requirement and attach passing evidence."
+}

chapter5/agent-creator/reference_agent/domain_tools.py

@@ -0,0 +1,88 @@
+"""Deterministic policy-record adapter configured by ``domain_spec.json``."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+ROOT = Path(__file__).resolve().parent
+
+
+def _spec() -> dict[str, Any]:
+    with (ROOT / "domain_spec.json").open(encoding="utf-8") as handle:
+        value = json.load(handle)
+    if not isinstance(value, dict):
+        raise ValueError("domain_spec.json must contain an object")
+    return value
+
+
+def evaluate_policy_records(records: list[dict[str, Any]]) -> dict[str, Any]:
+    spec = _spec()
+    required_field = spec["required_field"]
+    status_field = spec["status_field"]
+    identifier_field = spec["identifier_field"]
+    evidence_field = spec["evidence_field"]
+    passing = {str(value).casefold() for value in spec["passing_values"]}
+    remediation = {
+        str(key).casefold(): value
+        for key, value in spec["remediation_by_status"].items()
+    }
+    failures: list[dict[str, Any]] = []
+    normalized: list[dict[str, Any]] = []
+    for index, record in enumerate(records):
+        if not isinstance(record, dict):
+            raise ValueError(f"record {index} must be an object")
+        missing = [
+            field
+            for field in (identifier_field, required_field, status_field, evidence_field)
+            if field not in record
+        ]
+        if missing:
+            raise ValueError(f"record {index} missing fields: {', '.join(missing)}")
+        if not isinstance(record[required_field], bool):
+            raise ValueError(f"record {index} {required_field} must be boolean")
+        status = str(record[status_field])
+        row = {
+            "id": record[identifier_field],
+            "required": record[required_field],
+            "status": status,
+            "evidence": record[evidence_field],
+            "passed": status.casefold() in passing,
+        }
+        normalized.append(row)
+        if row["required"] and not row["passed"]:
+            failures.append(
+                {
+                    **row,
+                    "remediation": remediation.get(
+                        status.casefold(), spec["default_remediation"]
+                    ),
+                }
+            )
+    approved = not failures
+    return {
+        "approved": approved,
+        "decision": spec["approved_label"] if approved else spec["rejected_label"],
+        "evaluated_count": len(normalized),
+        "failed_required_count": len(failures),
+        "failed_required_records": failures,
+        "records": normalized,
+    }
+
+
+def execute_tool(name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+    spec = _spec()
+    if name == spec["tool_name"]:
+        records = arguments.get(spec["records_argument"])
+        if not isinstance(records, list) or not records:
+            return {
+                "ok": False,
+                "error": f"{spec['records_argument']} must be a non-empty array",
+            }
+        try:
+            return {"ok": True, "result": evaluate_policy_records(records)}
+        except (KeyError, TypeError, ValueError) as exc:
+            return {"ok": False, "error": str(exc)}
+    return {"ok": False, "error": f"unknown tool: {name}"}

chapter5/agent-creator/reference_agent/main.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import argparse
+import json
+
+from agent import GeneratedAgent
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run the generated Agent")
+    parser.add_argument("--task", required=True)
+    parser.add_argument("--model")
+    parser.add_argument("--history-json", default="[]")
+    args = parser.parse_args()
+    history = json.loads(args.history_json)
+    if not isinstance(history, list):
+        raise SystemExit("--history-json must decode to a list")
+    result = GeneratedAgent(model=args.model).run(args.task, history=history)
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    raise SystemExit(0 if result["ok"] else 1)
+
+
+if __name__ == "__main__":
+    main()

chapter5/agent-creator/reference_agent/requirements.txt

@@ -0,0 +1,2 @@
+openai>=1.30.0
+pytest>=7.0.0

chapter5/agent-creator/reference_agent/system_prompt.md

@@ -0,0 +1,9 @@
+You are a reliable, tool-using assistant.
+
+Follow these rules:
+
+1. Use tools whenever the answer depends on external or computed facts.
+2. Never invent a tool result. Wait for the tool response and cite it in the answer.
+3. Validate required arguments before calling a tool.
+4. If a tool fails, explain the failure and either correct the arguments or stop safely.
+5. Keep responses concise and explicitly distinguish observations from conclusions.

chapter5/agent-creator/reference_agent/tests/test_contract.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import json
+import copy
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from agent import GeneratedAgent
+
+
+class FakeCompletions:
+    def __init__(self):
+        self.calls = []
+
+    def create(self, **kwargs):
+        self.calls.append(copy.deepcopy(kwargs))
+        if len(self.calls) == 1:
+            tool_call = SimpleNamespace(
+                id="call-1",
+                function=SimpleNamespace(
+                    name="lookup_domain_fact",
+                    arguments=json.dumps({"query": "purpose"}),
+                ),
+            )
+            message = SimpleNamespace(content=None, tool_calls=[tool_call])
+        else:
+            assert kwargs["messages"][-1]["role"] == "tool"
+            message = SimpleNamespace(content="Verified answer", tool_calls=[])
+        usage = SimpleNamespace(prompt_tokens=10, completion_tokens=3)
+        return SimpleNamespace(choices=[SimpleNamespace(message=message)], usage=usage)
+
+
+def test_standard_tool_loop_keeps_assistant_call_and_tool_result():
+    completions = FakeCompletions()
+    client = SimpleNamespace(chat=SimpleNamespace(completions=completions))
+    result = GeneratedAgent(model="test-model", client=client).run("What is your purpose?")
+    assert result["ok"] is True
+    assert result["answer"] == "Verified answer"
+    second_messages = completions.calls[1]["messages"]
+    assert second_messages[-2]["role"] == "assistant"
+    assert second_messages[-2]["tool_calls"][0]["id"] == "call-1"
+    assert second_messages[-1]["role"] == "tool"
+    assert second_messages[-1]["tool_call_id"] == "call-1"
+    assert result["messages"][:-1] == second_messages
+    assert result["messages"][-1] == {
+        "role": "assistant",
+        "content": "Verified answer",
+    }
+    assert result["usage"] == {
+        "prompt_tokens": 20,
+        "cached_prompt_tokens": 0,
+        "completion_tokens": 6,
+        "requests": 2,
+    }
+
+
+def test_prior_multiturn_history_is_preserved_in_order():
+    completions = FakeCompletions()
+    client = SimpleNamespace(chat=SimpleNamespace(completions=completions))
+    history = [
+        {"role": "user", "content": "Remember owner Mei-Lin."},
+        {"role": "assistant", "content": "Owner Mei-Lin retained."},
+    ]
+
+    result = GeneratedAgent(model="test-model", client=client).run(
+        "Evaluate the release.", history=history
+    )
+
+    first_messages = completions.calls[0]["messages"]
+    assert first_messages[1:3] == history
+    assert result["messages"][1:3] == history

chapter5/agent-creator/reference_agent/tests/test_domain_policy.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from domain_tools import execute_tool
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def load_spec():
+    return json.loads((ROOT / "domain_spec.json").read_text(encoding="utf-8"))
+
+
+def record(spec, *, identifier, required, status, evidence):
+    return {
+        spec["identifier_field"]: identifier,
+        spec["required_field"]: required,
+        spec["status_field"]: status,
+        spec["evidence_field"]: evidence,
+    }
+
+
+def test_required_nonpassing_record_refuses_with_exact_evidence():
+    spec = load_spec()
+    records = [
+        record(
+            spec,
+            identifier="required-check",
+            required=True,
+            status="failed",
+            evidence="observed failure",
+        )
+    ]
+    result = execute_tool(spec["tool_name"], {spec["records_argument"]: records})
+    assert result["ok"] is True
+    assert result["result"]["approved"] is False
+    assert result["result"]["decision"] == spec["rejected_label"]
+    assert result["result"]["failed_required_records"][0]["evidence"] == "observed failure"
+
+
+def test_only_required_nonpassing_records_block_approval():
+    spec = load_spec()
+    passing = spec["passing_values"][0]
+    records = [
+        record(spec, identifier="required", required=True, status=passing, evidence="ok"),
+        record(spec, identifier="optional", required=False, status="failed", evidence="optional"),
+    ]
+    result = execute_tool(spec["tool_name"], {spec["records_argument"]: records})
+    assert result["ok"] is True
+    assert result["result"]["approved"] is True
+    assert result["result"]["decision"] == spec["approved_label"]
+
+
+def test_missing_or_empty_records_fail_closed():
+    spec = load_spec()
+    result = execute_tool(spec["tool_name"], {})
+    assert result["ok"] is False
+    assert spec["records_argument"] in result["error"]

chapter5/agent-creator/reference_agent/tools.json

@@ -0,0 +1,19 @@
+{
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "lookup_domain_fact",
+        "description": "Look up a fact in the Agent's verified domain knowledge base. Use this before answering domain-specific factual questions.",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "query": {"type": "string", "description": "A concise lookup query."}
+          },
+          "required": ["query"],
+          "additionalProperties": false
+        }
+      }
+    }
+  ]
+}

chapter5/agent-creator/requirements.txt

@@ -0,0 +1,3 @@
+openai>=1.30.0
+python-dotenv>=1.0.0
+pytest>=7.0.0

chapter5/agent-creator/test_creator.py

@@ -0,0 +1,360 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from creator import (
+    AgentCreator,
+    ResolvedBackend,
+    SCRATCH_FILE_GROUPS,
+    _usage_cost,
+    load_protocol,
+)
+from validator import _audit_case, _structural_check
+
+
+def response(payload):
+    message = SimpleNamespace(content=json.dumps(payload))
+    usage = SimpleNamespace(prompt_tokens=100, completion_tokens=200)
+    return SimpleNamespace(choices=[SimpleNamespace(message=message)], usage=usage)
+
+
+class FakeCompletions:
+    def __init__(self, payload):
+        self.payload = payload
+
+    def create(self, **_kwargs):
+        return response(self.payload)
+
+
+class SequenceCompletions:
+    def __init__(self, payloads):
+        self.payloads = iter(payloads)
+
+    def create(self, **_kwargs):
+        payload = next(self.payloads)
+        if isinstance(payload, str):
+            message = SimpleNamespace(content=payload)
+            usage = SimpleNamespace(prompt_tokens=10, completion_tokens=20)
+            return SimpleNamespace(choices=[SimpleNamespace(message=message)], usage=usage)
+        return response(payload)
+
+
+def fake_client(payload):
+    return SimpleNamespace(chat=SimpleNamespace(completions=FakeCompletions(payload)))
+
+
+def sequence_client(payloads):
+    return SimpleNamespace(chat=SimpleNamespace(completions=SequenceCompletions(payloads)))
+
+
+def template_payload():
+    return {
+        "specialization": {
+            "name": "test-agent",
+            "role": "Evaluate required test checks from supplied evidence.",
+            "sample_task": "evaluate the checks",
+            "tool_name": "evaluate_test_checks",
+            "tool_description": "Evaluate every supplied test check.",
+            "record_noun": "test check",
+            "records_argument": "checks",
+            "identifier_field": "id",
+            "required_field": "required",
+            "status_field": "outcome",
+            "evidence_field": "evidence",
+            "passing_values": ["passed"],
+            "approved_label": "APPROVED",
+            "rejected_label": "REFUSED",
+            "remediation_by_status": {"failed": "Fix and rerun the check."},
+            "default_remediation": "Resolve the check and attach passing evidence.",
+        }
+    }
+
+
+def scratch_blueprint():
+    return {
+        "name": "release-agent",
+        "sample_task": "evaluate supplied release gates",
+        "design": {
+            "tool_name": "evaluate_gates",
+            "records_argument": "gates",
+            "identifier_field": "id",
+            "required_field": "required",
+            "status_field": "outcome",
+            "evidence_field": "evidence",
+            "passing_value": "passed",
+            "agent_contract": "bounded standard tool loop",
+            "dispatcher_contract": "evaluate every required gate",
+            "cli_contract": "accept --task and --model and print JSON",
+            "test_contract": "test refusal and tool message preservation",
+        },
+    }
+
+
+def test_staged_scratch_generation_collects_every_file_and_call(tmp_path: Path):
+    group_payloads = [
+        {
+            "files": {
+                path: ('{"tools": []}' if path == "tools.json" else "content")
+                for path in group
+            }
+        }
+        for group in SCRATCH_FILE_GROUPS
+    ]
+    creator = AgentCreator(
+        sequence_client([scratch_blueprint(), *group_payloads]), "test-model"
+    )
+
+    blueprint, files, stats = creator._generate_scratch_files(
+        "make a release agent", tmp_path / "scratch-checkpoint"
+    )
+
+    assert blueprint["design"]["tool_name"] == "evaluate_gates"
+    assert set(files) == {path for group in SCRATCH_FILE_GROUPS for path in group}
+    assert stats.model_calls == 1 + len(SCRATCH_FILE_GROUPS)
+    assert stats.prompt_tokens == 100 * (1 + len(SCRATCH_FILE_GROUPS))
+    assert stats.completion_tokens == 200 * (1 + len(SCRATCH_FILE_GROUPS))
+
+
+def test_scratch_creation_recovers_only_empty_staging_directory(tmp_path: Path):
+    output = tmp_path / "scratch"
+    output.mkdir()
+    group_payloads = [
+        {
+            "files": {
+                path: ('{"tools": []}' if path == "tools.json" else "content")
+                for path in group
+            }
+        }
+        for group in SCRATCH_FILE_GROUPS
+    ]
+    creator = AgentCreator(
+        sequence_client([scratch_blueprint(), *group_payloads]), "test-model"
+    )
+    creator._repair_until_deterministic = lambda **kwargs: kwargs["stats"]
+
+    stats = creator.create_from_scratch("make a release agent", output)
+
+    assert stats.strategy == "scratch"
+    assert (output / "generation.json").is_file()
+
+
+def test_scratch_creation_preserves_nonempty_existing_output(tmp_path: Path):
+    output = tmp_path / "scratch"
+    output.mkdir()
+    (output / "user-file.txt").write_text("preserve", encoding="utf-8")
+    creator = AgentCreator(fake_client({}), "test-model")
+
+    with pytest.raises(FileExistsError):
+        creator.create_from_scratch("make a release agent", output)
+
+    assert (output / "user-file.txt").read_text(encoding="utf-8") == "preserve"
+
+
+def test_template_mode_copies_core_and_applies_specialization(tmp_path: Path):
+    creator = AgentCreator(fake_client(template_payload()), "test-model")
+    output = tmp_path / "agent"
+    stats = creator.create_from_template("make a test agent", output)
+    assert stats.strategy == "template"
+    assert (output / "agent.py").is_file()
+    assert (output / "tests/test_contract.py").is_file()
+    assert "Never invent a registration ID" in (output / "system_prompt.md").read_text()
+    assert json.loads((output / "domain_spec.json").read_text())["records_argument"] == "checks"
+
+
+def test_normalizes_bare_tool_array():
+    raw = {"tools.json": json.dumps([{"type": "function", "function": {"name": "x"}}])}
+    normalized = AgentCreator._normalize_files(raw)
+    assert json.loads(normalized["tools.json"])["tools"][0]["function"]["name"] == "x"
+
+
+def test_ask_retries_truncated_json_and_accounts_for_both_real_calls():
+    creator = AgentCreator(
+        sequence_client(['{"specialization":{"name":"unterminated', template_payload()]),
+        "test-model",
+    )
+
+    payload, stats = creator._ask("return a specialization")
+
+    assert payload == template_payload()
+    assert stats.model_calls == 2
+    assert stats.prompt_tokens == 110
+    assert stats.completion_tokens == 220
+
+
+def test_rejects_path_traversal(tmp_path: Path):
+    with pytest.raises(ValueError, match="disallowed"):
+        AgentCreator._safe_files({"files": {"../escape.py": "bad"}}, {"domain_spec.json"})
+
+
+def test_safe_files_accepts_direct_allowlisted_mapping_and_structured_json():
+    files = AgentCreator._safe_files(
+        {
+            "domain_tools.py": "def evaluate():\n    return True\n",
+            "tools.json": {"tools": []},
+        },
+        {"domain_tools.py", "tools.json"},
+    )
+
+    assert files["domain_tools.py"].startswith("def evaluate")
+    assert json.loads(files["tools.json"]) == {"tools": []}
+
+
+@pytest.mark.parametrize("wrapper", ["artifacts", "outputs", "generated_files"])
+def test_safe_files_accepts_one_known_wrapper_without_relaxing_paths(wrapper: str):
+    files = AgentCreator._safe_files(
+        {wrapper: {"domain_tools.py": "def evaluate():\n    return True\n"}},
+        {"domain_tools.py"},
+    )
+    assert set(files) == {"domain_tools.py"}
+
+    with pytest.raises(ValueError, match="disallowed"):
+        AgentCreator._safe_files(
+            {wrapper: {"../escape.py": "bad"}}, {"domain_tools.py"}
+        )
+
+
+def test_safe_files_does_not_treat_arbitrary_payload_as_file_mapping():
+    with pytest.raises(ValueError, match="files object"):
+        AgentCreator._safe_files(
+            {"name": "not-a-file-envelope", "domain_tools.py": "content"},
+            {"domain_tools.py"},
+        )
+
+
+def test_resolved_backend_aliases_real_endpoint_for_generated_agents():
+    backend = ResolvedBackend(
+        provider="moonshot",
+        client=object(),
+        model="kimi-k3",
+        api_key="test-key-not-a-secret",
+        base_url="https://api.moonshot.cn/v1",
+    )
+    env = backend.generated_agent_env()
+    assert env["OPENAI_API_KEY"] == "test-key-not-a-secret"
+    assert env["OPENAI_BASE_URL"] == "https://api.moonshot.cn/v1"
+    assert env["OPENAI_MODEL"] == "kimi-k3"
+    assert env["OPENROUTER_API_KEY"] == ""
+
+
+def test_structural_gate_requires_common_live_cli(tmp_path: Path):
+    root = tmp_path / "generated"
+    root.mkdir()
+    for relative in (
+        "agent.py", "domain_tools.py", "system_prompt.md", "requirements.txt"
+    ):
+        (root / relative).write_text("", encoding="utf-8")
+    (root / "main.py").write_text(
+        "import argparse\nparser = argparse.ArgumentParser()\n"
+        "parser.add_argument('--facts')\n",
+        encoding="utf-8",
+    )
+    (root / "tools.json").write_text('{"tools": []}', encoding="utf-8")
+    tests = root / "tests"
+    tests.mkdir()
+    (tests / "test_contract.py").write_text("def test_placeholder(): pass\n", encoding="utf-8")
+
+    ok, errors = _structural_check(root)
+
+    assert ok is False
+    assert "main.py must implement the common live CLI option --task" in errors
+    assert "main.py must implement the common live CLI option --model" in errors
+
+
+def test_frozen_protocol_has_three_common_cases_and_native_pricing():
+    protocol, digest = load_protocol()
+
+    assert len(digest) == 64
+    assert [case["kind"] for case in protocol["live_cases"]].count("basic_task") == 2
+    assert [case["kind"] for case in protocol["live_cases"]].count("multi_turn_state") == 1
+    assert protocol["backend_requirement"]["model"] == "kimi-k3"
+    assert protocol["backend_requirement"]["pricing"]["currency"] == "CNY"
+
+
+def test_native_cost_uses_observed_cached_split():
+    protocol, _digest = load_protocol()
+    cost = _usage_cost(
+        {
+            "prompt_tokens": 1000,
+            "cached_prompt_tokens": 400,
+            "completion_tokens": 100,
+            "requests": 2,
+        },
+        protocol["backend_requirement"]["pricing"],
+    )
+
+    assert cost["uncached_prompt_tokens"] == 600
+    assert cost["cost"] == pytest.approx(0.0228)
+    assert cost["currency"] == "CNY"
+
+
+def test_case_audit_requires_matching_tool_protocol_history_usage_and_evidence():
+    case = {
+        "id": "stateful",
+        "kind": "multi_turn_state",
+        "history": [
+            {"role": "user", "content": "Remember Mei-Lin."},
+            {"role": "assistant", "content": "Remembered Mei-Lin."},
+        ],
+        "task": "Evaluate rollback_drill.",
+        "expected": {
+            "decision": "REFUSED",
+            "failed_ids": ["rollback_drill"],
+            "evidence": ["too slow"],
+            "answer_substrings": ["REFUSED", "rollback_drill"],
+            "forbidden_answer_substrings": ["APPROVED"],
+            "context_markers": ["Mei-Lin"],
+        },
+    }
+    result = {
+        "ok": True,
+        "answer": "REFUSED for rollback_drill. Owner Mei-Lin must rerun it.",
+        "messages": [
+            {"role": "system", "content": "system"},
+            *case["history"],
+            {"role": "user", "content": case["task"]},
+            {
+                "role": "assistant",
+                "content": "",
+                "tool_calls": [
+                    {
+                        "id": "call-1",
+                        "type": "function",
+                        "function": {"name": "evaluate", "arguments": "{}"},
+                    }
+                ],
+            },
+            {
+                "role": "tool",
+                "tool_call_id": "call-1",
+                "content": json.dumps(
+                    {
+                        "decision": "REFUSED",
+                        "failed": "rollback_drill",
+                        "evidence": "too slow",
+                    }
+                ),
+            },
+            {"role": "assistant", "content": "REFUSED"},
+        ],
+        "usage": {
+            "prompt_tokens": 100,
+            "cached_prompt_tokens": 0,
+            "completion_tokens": 20,
+            "requests": 2,
+        },
+    }
+
+    audit = _audit_case(
+        case,
+        process_ok=True,
+        result=result,
+        elapsed_s=1.0,
+        extra_env={"OPENAI_API_KEY": "credential-not-in-evidence"},
+    )
+
+    assert audit["passed"] is True
+    assert audit["score"] == audit["max_score"]

chapter5/agent-creator/validator.py

@@ -0,0 +1,448 @@
+"""Fixed structural and real-run validation for Experiment 5-13."""
+
+from __future__ import annotations
+
+import ast
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+REQUIRED_FILES = {
+    "agent.py",
+    "domain_tools.py",
+    "main.py",
+    "system_prompt.md",
+    "tools.json",
+    "requirements.txt",
+    "tests/test_contract.py",
+}
+
+
+@dataclass
+class ValidationReport:
+    structural_ok: bool
+    compile_ok: bool
+    tests_ok: bool
+    live_ok: bool | None
+    protocol_ok: bool | None
+    multiturn_ok: bool | None
+    raw_evidence_ok: bool | None
+    usage_ok: bool | None
+    semantic_ok: bool | None
+    duration_s: float
+    errors: list[str]
+    live_result: dict[str, Any] | None = None
+    semantic_judgment: dict[str, Any] | None = None
+    live_cases: list[dict[str, Any]] = field(default_factory=list)
+    quality_score: int = 0
+    quality_max_score: int = 0
+
+    @property
+    def ok(self) -> bool:
+        optional_gates = (
+            self.live_ok,
+            self.protocol_ok,
+            self.multiturn_ok,
+            self.raw_evidence_ok,
+            self.usage_ok,
+            self.semantic_ok,
+        )
+        return (
+            self.structural_ok
+            and self.compile_ok
+            and self.tests_ok
+            and all(value is not False for value in optional_gates)
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["ok"] = self.ok
+        return payload
+
+
+def _attribute_name(node: ast.AST) -> str:
+    parts: list[str] = []
+    current = node
+    while isinstance(current, ast.Attribute):
+        parts.append(current.attr)
+        current = current.value
+    if isinstance(current, ast.Name):
+        parts.append(current.id)
+    return ".".join(reversed(parts))
+
+
+def _structural_check(root: Path) -> tuple[bool, list[str]]:
+    errors: list[str] = []
+    missing = sorted(path for path in REQUIRED_FILES if not (root / path).is_file())
+    if missing:
+        errors.append(f"missing required files: {', '.join(missing)}")
+    trees: dict[str, ast.AST] = {}
+    sources: dict[str, str] = {}
+    for relative in ("agent.py", "domain_tools.py", "main.py"):
+        path = root / relative
+        if path.exists():
+            sources[relative] = path.read_text(encoding="utf-8")
+            try:
+                trees[relative] = ast.parse(sources[relative], filename=str(path))
+            except SyntaxError as exc:
+                errors.append(f"{relative}: {exc}")
+    tools_path = root / "tools.json"
+    if tools_path.exists():
+        try:
+            tools = json.loads(tools_path.read_text(encoding="utf-8"))["tools"]
+            names = [tool["function"]["name"] for tool in tools]
+            if not names or len(names) != len(set(names)):
+                errors.append("tools.json must contain unique function names")
+        except (KeyError, TypeError, json.JSONDecodeError) as exc:
+            errors.append(f"invalid tools.json: {exc}")
+
+    agent_tree = trees.get("agent.py")
+    if agent_tree is not None:
+        string_constants = {
+            node.value
+            for node in ast.walk(agent_tree)
+            if isinstance(node, ast.Constant) and isinstance(node.value, str)
+        }
+        attributes = {
+            _attribute_name(node)
+            for node in ast.walk(agent_tree)
+            if isinstance(node, ast.Attribute)
+        }
+        for marker in ("assistant", "tool", "tool_call_id", "tool_calls"):
+            if marker not in string_constants and not any(
+                name.endswith(f".{marker}") for name in attributes
+            ):
+                errors.append(f"agent loop missing required protocol element: {marker}")
+        run_functions = [
+            node
+            for node in ast.walk(agent_tree)
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "run"
+        ]
+        run_args = {
+            arg.arg
+            for function in run_functions
+            for arg in (*function.args.args, *function.args.kwonlyargs)
+        }
+        if "history" not in run_args:
+            errors.append("Agent run contract must accept prior multi-turn history")
+        has_bound = any(
+            isinstance(node, ast.arg) and node.arg in {"max_iterations", "max_steps", "max_turns"}
+            for node in ast.walk(agent_tree)
+        )
+        has_bounded_loop = any(
+            isinstance(node, ast.For)
+            and isinstance(node.iter, ast.Call)
+            and isinstance(node.iter.func, ast.Name)
+            and node.iter.func.id == "range"
+            for node in ast.walk(agent_tree)
+        )
+        if not (has_bound and has_bounded_loop):
+            errors.append(
+                "agent loop must expose a maximum-iteration bound and use a bounded for/range loop"
+            )
+        if not any(
+            isinstance(node, ast.ImportFrom)
+            and node.module == "openai"
+            and any(alias.name == "OpenAI" for alias in node.names)
+            for node in ast.walk(agent_tree)
+        ):
+            errors.append("agent.py must use the current OpenAI client class")
+        if not any(name.endswith("chat.completions.create") for name in attributes):
+            errors.append("agent.py must use the current chat.completions API")
+        for evidence_key in ("messages", "usage"):
+            if evidence_key not in string_constants:
+                errors.append(f"live result must preserve raw {evidence_key} evidence")
+
+    main_tree = trees.get("main.py")
+    if main_tree is not None:
+        main_strings = {
+            node.value
+            for node in ast.walk(main_tree)
+            if isinstance(node, ast.Constant) and isinstance(node.value, str)
+        }
+        for option in ("--task", "--model", "--history-json"):
+            if option not in main_strings:
+                errors.append(f"main.py must implement the common live CLI option {option}")
+
+    for file in root.rglob("*"):
+        if file.is_file() and file.name != ".env.example":
+            text = file.read_text(encoding="utf-8", errors="ignore")
+            if re.search(r"\bsk-[A-Za-z0-9_-]{12,}\b", text):
+                errors.append(f"possible embedded secret in {file.relative_to(root)}")
+    return not errors, errors
+
+
+def _run(
+    command: list[str],
+    root: Path,
+    timeout: int,
+    extra_env: dict[str, str] | None = None,
+) -> tuple[bool, str, float]:
+    started = time.perf_counter()
+    # Generated Agents live below ``runs/`` while the experiment's repository-
+    # level pytest.ini is intentionally discovered from a parent directory.
+    # Pytest therefore does not reliably prepend the generated Agent root to
+    # sys.path.  Make the executable-under-test importable exactly as it is when
+    # launched via ``python main.py``; otherwise valid ``import agent`` and
+    # ``import domain_tools`` statements fail during collection before a single
+    # generated test can run.
+    inherited_pythonpath = os.environ.get("PYTHONPATH", "")
+    pythonpath = str(root)
+    if inherited_pythonpath:
+        pythonpath += os.pathsep + inherited_pythonpath
+    proc = subprocess.run(
+        command,
+        cwd=root,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        timeout=timeout,
+        env={
+            **os.environ,
+            **(extra_env or {}),
+            "PYTHONDONTWRITEBYTECODE": "1",
+            "PYTHONPATH": pythonpath,
+        },
+    )
+    return proc.returncode == 0, proc.stdout[-100000:], round(time.perf_counter() - started, 3)
+
+
+def _contains_all(text: str, expected: list[str]) -> bool:
+    folded = text.casefold()
+    return all(value.casefold() in folded for value in expected)
+
+
+def _history_is_preserved(messages: list[Any], history: list[dict[str, Any]]) -> bool:
+    if not history:
+        return True
+    cursor = 0
+    for message in messages:
+        if cursor >= len(history) or not isinstance(message, dict):
+            continue
+        expected = history[cursor]
+        if message.get("role") == expected["role"] and message.get("content") == expected["content"]:
+            cursor += 1
+    return cursor == len(history)
+
+
+def _protocol_is_valid(messages: list[Any]) -> tuple[bool, list[str], str]:
+    assistant_ids: list[str] = []
+    tool_ids: list[str] = []
+    tool_texts: list[str] = []
+    for message in messages:
+        if not isinstance(message, dict):
+            continue
+        if message.get("role") == "assistant":
+            for call in message.get("tool_calls") or []:
+                if isinstance(call, dict) and isinstance(call.get("id"), str):
+                    assistant_ids.append(call["id"])
+        if message.get("role") == "tool":
+            if isinstance(message.get("tool_call_id"), str):
+                tool_ids.append(message["tool_call_id"])
+            tool_texts.append(str(message.get("content", "")))
+    valid = bool(assistant_ids) and assistant_ids == tool_ids
+    return valid, assistant_ids, "\n".join(tool_texts)
+
+
+def _usage_is_complete(result: dict[str, Any]) -> bool:
+    usage = result.get("usage")
+    return (
+        isinstance(usage, dict)
+        and isinstance(usage.get("prompt_tokens"), int)
+        and usage["prompt_tokens"] > 0
+        and isinstance(usage.get("completion_tokens"), int)
+        and usage["completion_tokens"] > 0
+        and isinstance(usage.get("requests"), int)
+        and usage["requests"] > 0
+    )
+
+
+def _credential_free(payload: Any, extra_env: dict[str, str] | None) -> bool:
+    text = json.dumps(payload, ensure_ascii=False)
+    if re.search(r"\bsk-[A-Za-z0-9_-]{12,}\b", text):
+        return False
+    for name, value in (extra_env or {}).items():
+        if "KEY" in name and value and len(value) >= 8 and value in text:
+            return False
+    return True
+
+
+def _audit_case(
+    case: dict[str, Any],
+    *,
+    process_ok: bool,
+    result: dict[str, Any],
+    elapsed_s: float,
+    extra_env: dict[str, str] | None,
+) -> dict[str, Any]:
+    expected = case["expected"]
+    answer = str(result.get("answer", ""))
+    messages = result.get("messages")
+    messages_list = messages if isinstance(messages, list) else []
+    protocol_valid, tool_call_ids, tool_text = _protocol_is_valid(messages_list)
+    expected_failed_ids = list(expected.get("failed_ids") or [])
+    expected_evidence = list(expected.get("evidence") or [])
+    checks = {
+        "process_exit_zero": process_ok,
+        "agent_reported_ok": result.get("ok") is True,
+        "answer_has_expected_decision": expected["decision"].casefold() in answer.casefold(),
+        "answer_has_required_content": _contains_all(
+            answer, list(expected.get("answer_substrings") or [])
+        ),
+        "answer_avoids_forbidden_content": not any(
+            value.casefold() in answer.casefold()
+            for value in expected.get("forbidden_answer_substrings") or []
+        ),
+        "standard_tool_protocol": protocol_valid,
+        "tool_result_has_expected_decision": expected["decision"].casefold()
+        in tool_text.casefold(),
+        "tool_result_covers_failed_ids": _contains_all(tool_text, expected_failed_ids),
+        "tool_result_covers_evidence": _contains_all(tool_text, expected_evidence),
+        "history_preserved": _history_is_preserved(messages_list, case.get("history") or []),
+        "context_used_in_answer": _contains_all(
+            answer, list(expected.get("context_markers") or [])
+        ),
+        "provider_usage_present": _usage_is_complete(result),
+        "raw_evidence_credential_free": _credential_free(result, extra_env),
+    }
+    return {
+        "id": case["id"],
+        "kind": case["kind"],
+        "task": case["task"],
+        "history": case.get("history") or [],
+        "expected": expected,
+        "process_elapsed_s": elapsed_s,
+        "checks": checks,
+        "score": sum(checks.values()),
+        "max_score": len(checks),
+        "passed": all(checks.values()),
+        "tool_call_ids": tool_call_ids,
+        "raw_result": result,
+    }
+
+
+def validate_agent(
+    root: Path,
+    *,
+    live_task: str | None = None,
+    live_cases: list[dict[str, Any]] | None = None,
+    model: str | None = None,
+    timeout: int = 180,
+    extra_env: dict[str, str] | None = None,
+) -> ValidationReport:
+    started = time.perf_counter()
+    structural_ok, errors = _structural_check(root)
+    compile_ok, compile_output, _compile_s = _run(
+        [sys.executable, "-m", "compileall", "-q", "agent.py", "domain_tools.py", "main.py"],
+        root,
+        timeout,
+        extra_env,
+    )
+    if not compile_ok:
+        errors.append(f"compile failed:\n{compile_output}")
+    tests_ok, test_output, _test_s = _run(
+        [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider"],
+        root,
+        timeout,
+        extra_env,
+    )
+    if not tests_ok:
+        errors.append(f"tests failed:\n{test_output}")
+
+    cases = list(live_cases or [])
+    if live_task is not None and not cases:
+        cases = [
+            {
+                "id": "diagnostic_live_task",
+                "kind": "basic_task",
+                "history": [],
+                "task": live_task,
+                "expected": {
+                    "decision": "",
+                    "failed_ids": [],
+                    "evidence": [],
+                    "answer_substrings": [],
+                    "forbidden_answer_substrings": [],
+                    "context_markers": [],
+                },
+            }
+        ]
+
+    audited_cases: list[dict[str, Any]] = []
+    if cases and structural_ok and compile_ok and tests_ok:
+        for case in cases:
+            command = [sys.executable, "main.py", "--task", case["task"]]
+            if model:
+                command += ["--model", model]
+            history = case.get("history") or []
+            command += ["--history-json", json.dumps(history, ensure_ascii=False)]
+            process_ok, output, elapsed_s = _run(command, root, timeout, extra_env)
+            try:
+                result = json.loads(output)
+                if not isinstance(result, dict):
+                    raise ValueError("CLI result must be an object")
+            except (json.JSONDecodeError, ValueError) as exc:
+                result = {
+                    "ok": False,
+                    "answer": "",
+                    "raw_stdout": output,
+                    "parse_error": f"{type(exc).__name__}: {exc}",
+                }
+            audited = _audit_case(
+                case,
+                process_ok=process_ok,
+                result=result,
+                elapsed_s=elapsed_s,
+                extra_env=extra_env,
+            )
+            audited_cases.append(audited)
+            if not audited["passed"]:
+                failed = [name for name, value in audited["checks"].items() if not value]
+                errors.append(f"live case {case['id']} failed checks: {', '.join(failed)}")
+    elif cases:
+        errors.append("live cases skipped because deterministic gates failed")
+
+    live_requested = bool(cases)
+    live_ok = all(case["passed"] for case in audited_cases) and len(audited_cases) == len(cases) if live_requested else None
+    protocol_ok = all(case["checks"]["standard_tool_protocol"] for case in audited_cases) if live_requested else None
+    state_cases = [case for case in audited_cases if case["kind"] == "multi_turn_state"]
+    multiturn_ok = (
+        bool(state_cases)
+        and all(
+            case["checks"]["history_preserved"] and case["checks"]["context_used_in_answer"]
+            for case in state_cases
+        )
+        if live_requested
+        else None
+    )
+    raw_evidence_ok = all(
+        case["checks"]["raw_evidence_credential_free"] for case in audited_cases
+    ) if live_requested else None
+    usage_ok = all(case["checks"]["provider_usage_present"] for case in audited_cases) if live_requested else None
+    quality_score = sum(case["score"] for case in audited_cases)
+    quality_max = sum(case["max_score"] for case in audited_cases)
+    live_result = audited_cases[0]["raw_result"] if len(audited_cases) == 1 else None
+    return ValidationReport(
+        structural_ok=structural_ok,
+        compile_ok=compile_ok,
+        tests_ok=tests_ok,
+        live_ok=live_ok,
+        protocol_ok=protocol_ok,
+        multiturn_ok=multiturn_ok,
+        raw_evidence_ok=raw_evidence_ok,
+        usage_ok=usage_ok,
+        semantic_ok=live_ok,
+        duration_s=round(time.perf_counter() - started, 3),
+        errors=errors,
+        live_result=live_result,
+        live_cases=audited_cases,
+        quality_score=quality_score,
+        quality_max_score=quality_max,
+    )