ai-agent-book/chapter10/talkact-reproduction/validate_campaign.py

#!/usr/bin/env python3
"""Validate the retained historical 10-3 campaign for current Experiment 10-3."""

from __future__ import annotations

import argparse
import hashlib
import json
import re
from collections import Counter
from pathlib import Path


TASKS = (
    "forms-insurance",
    "booking-flight",
    "webmail-report",
    "meeting-helper",
)
CONDITIONS = ("duplex", "strawman")
SEEDS = (0, 1)
SOURCE_COMMIT = "7d70007f72d45ddfc1a14e8e229b6d444e4919a2"
SLOW_MODEL = "claude-opus-4-8"
FAST_MODEL = "claude-haiku-4-5"
USER_MODEL = "anthropic:claude-sonnet-4-5-20250929"


def sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def positive_usage(usage: dict | None) -> bool:
    return bool(usage) and sum(v for v in usage.values() if isinstance(v, int)) > 0


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("run_dir", type=Path)
    args = parser.parse_args()
    run_dir = args.run_dir.resolve()
    protocol = json.loads((run_dir / "protocol.json").read_text(encoding="utf-8"))
    summary = json.loads((run_dir / "summary.json").read_text(encoding="utf-8"))
    aggregate = json.loads((run_dir / "aggregate.json").read_text(encoding="utf-8"))

    expected_names = {
        f"{task}__{condition}__s{seed}.json"
        for task in TASKS
        for condition in CONDITIONS
        for seed in SEEDS
    }
    episode_paths = sorted((run_dir / "episodes").glob("*.json"))
    actual_names = {path.name for path in episode_paths}
    episodes = [json.loads(path.read_text(encoding="utf-8")) for path in episode_paths]

    episode_pairs = Counter((ep.get("task"), ep.get("condition")) for ep in episodes)
    event_kinds = Counter(
        event.get("kind") for episode in episodes for event in episode.get("events", [])
    )
    duplex_events = [
        event
        for episode in episodes
        if episode.get("condition") == "duplex"
        for event in episode.get("events", [])
    ]
    fast_to_slow = sum(
        len(event.get("to_slow", []))
        for event in duplex_events
        if event.get("kind") == "fast_turn"
    )
    slow_to_fast = sum(
        1
        for event in duplex_events
        if event.get("kind") in {"slow_ask_user", "slow_tell_user"}
    )
    latency_samples = sum(
        1
        for episode in episodes
        for event in episode.get("events", [])
        if event.get("kind") == "voice_latency"
    )
    provider_errors = [
        event
        for episode in episodes
        for event in episode.get("events", [])
        if event.get("kind")
        in {"episode_error", "slow_task_error", "episode_crashed"}
    ]

    gates = {
        "source_commit": protocol.get("source_commit") == SOURCE_COMMIT,
        "exact_campaign_shape": actual_names == expected_names and len(episodes) == 16,
        "two_seeds_per_task_condition": all(
            episode_pairs[(task, condition)] == 2
            for task in TASKS
            for condition in CONDITIONS
        ),
        "summary_has_16_fresh_rows": len(summary) == 16
        and Counter((row.get("task"), row.get("condition")) for row in summary)
        == episode_pairs,
        "all_episode_records_complete": all(
            ep.get("error") is None
            and isinstance(ep.get("env_state"), dict)
            and isinstance(ep.get("events"), list)
            and isinstance(ep.get("transcript"), list)
            for ep in episodes
        ),
        "anthropic_slow_model": all(ep.get("slow_model") == SLOW_MODEL for ep in episodes),
        "anthropic_fast_model": all(
            (ep.get("fast_model") == FAST_MODEL if ep.get("condition") == "duplex" else ep.get("fast_model") is None)
            for ep in episodes
        ),
        "explicit_anthropic_caller_override": all(
            ep.get("user_model") == USER_MODEL for ep in episodes
        )
        and protocol.get("caller_deviation", {}).get("model")
        == "claude-sonnet-4-5-20250929",
        "provider_usage_retained": all(
            positive_usage(ep.get("slow_usage"))
            and (
                positive_usage(ep.get("fast_usage"))
                if ep.get("condition") == "duplex"
                else ep.get("fast_usage") is None
            )
            for ep in episodes
        ),
        "no_runtime_provider_errors": not provider_errors,
        "real_concurrent_duplex_events": event_kinds["fast_turn"] > 0
        and event_kinds["slow_step"] > 0,
        "bidirectional_bridge_observed": fast_to_slow > 0 and slow_to_fast > 0,
        "voice_latency_samples": latency_samples > 0
        and all(
            any(event.get("kind") == "voice_latency" for event in ep.get("events", []))
            for ep in episodes
        ),
        "deterministic_checks_retained": all(
            "success" in row and "partial" in row and "details" in row
            for row in summary
        ),
        "judge_results_retained": all(
            "judge_error" not in row for row in summary
        ),
        "duplex_strawman_aggregate": {
            row.get("condition"): row.get("n_episodes") for row in aggregate
        }
        == {"duplex": 8, "strawman": 8},
        "no_credential_material": not any(
            re.search(rb"sk-ant-[A-Za-z0-9_-]{20,}|AIza[A-Za-z0-9_-]{20,}", path.read_bytes())
            for path in run_dir.rglob("*")
            if path.is_file()
        ),
    }

    files = []
    for path in sorted(p for p in run_dir.rglob("*") if p.is_file()):
        if path.name in {"acceptance.json", "manifest.json"}:
            continue
        files.append(
            {
                "path": str(path.relative_to(run_dir)),
                "bytes": path.stat().st_size,
                "sha256": sha256(path),
            }
        )

    acceptance = {
        "schema_version": 1,
        "experiment": "10-3",
        "run_dir": run_dir.name,
        "passed": all(gates.values()),
        "gates": gates,
        "counts": {
            "episodes": len(episodes),
            "duplex": sum(ep.get("condition") == "duplex" for ep in episodes),
            "strawman": sum(ep.get("condition") == "strawman" for ep in episodes),
            "fast_to_slow_relays": fast_to_slow,
            "slow_to_fast_events": slow_to_fast,
            "voice_latency_samples": latency_samples,
            "files_hashed": len(files),
        },
        "comparison": aggregate,
    }
    manifest = {
        "schema_version": 1,
        "experiment": "10-3",
        "run_dir": run_dir.name,
        "files": files,
    }
    (run_dir / "acceptance.json").write_text(json.dumps(acceptance, indent=2) + "\n", encoding="utf-8")
    (run_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(acceptance, indent=2))
    return 0 if acceptance["passed"] else 1


if __name__ == "__main__":
    raise SystemExit(main())