ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter6/controllable-tts/validate_artifacts.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""Validate real Fish S1 Experiment 6-6 media without making new API calls."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import platform
+import subprocess
+from datetime import datetime, timezone
+from pathlib import Path
+
+from evaluate_audio_quality import validate_study
+
+HERE = Path(__file__).parent
+MANIFEST = HERE / "reference_audio" / "manifest.json"
+RUN = HERE / "validation" / "latest.json"
+QUALITY_STUDY = HERE / "validation" / "audio_quality_study.json"
+
+
+def sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def probe(path: Path) -> dict[str, float | int | str]:
+    raw = subprocess.check_output([
+        "ffprobe", "-v", "error", "-show_entries", "format=duration,size,format_name",
+        "-of", "json", str(path),
+    ], text=True)
+    info = json.loads(raw)["format"]
+    return {"duration_seconds": float(info["duration"]), "size_bytes": int(info["size"]), "format": info["format_name"]}
+
+
+def main() -> int:
+    manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
+    run = json.loads(RUN.read_text(encoding="utf-8"))
+    profiles = manifest["profiles"]
+    reference_checks = []
+    for key, profile in sorted(profiles.items()):
+        path = HERE / "reference_audio" / profile["path"]
+        media = probe(path)
+        reference_checks.append({
+            "profile": key,
+            "path": str(path.relative_to(HERE)),
+            "exists": path.exists(),
+            "sha256": sha256(path),
+            "manifest_sha256": profile["sha256"],
+            "hash_matches": sha256(path) == profile["sha256"],
+            **media,
+        })
+    outputs = {}
+    for name, recorded in run["outputs"].items():
+        path = HERE / "output" / Path(recorded["path"]).name
+        outputs[name] = {
+            "path": str(path.relative_to(HERE)),
+            "sha256": sha256(path),
+            **probe(path),
+        }
+    dimensions = {
+        (profile["emotion"], profile["speed"], profile["style"])
+        for profile in profiles.values()
+    }
+    c_segments = run["outputs"]["C_24_reference_library"]["segments"]
+    routed_profiles = [segment["profile"] for segment in c_segments if segment.get("type") == "speech"]
+    required_routes = {"happy_fast_casual", "thinking_slow_formal", "neutral_normal_formal"}
+    gates = {
+        "fish_s1_provider_recorded": run.get("provider") == "Fish Audio" and run.get("backend") == "s1",
+        "same_authorized_source_reference": bool(manifest.get("source_reference_id")),
+        "exact_4x3x2_reference_library": len(profiles) == 24 and len(dimensions) == 24,
+        "all_reference_hashes_match": all(item["hash_matches"] for item in reference_checks),
+        "references_approximately_five_seconds": all(3.0 <= item["duration_seconds"] <= 7.0 for item in reference_checks),
+        "three_real_comparison_outputs": set(outputs) == {
+            "A_no_control_markers", "B_single_reference", "C_24_reference_library"
+        } and all(item["size_bytes"] > 1000 and item["duration_seconds"] > 0 for item in outputs.values()),
+        "required_marker_routes_exercised": required_routes.issubset(set(routed_profiles)),
+        "thinking_pause_1_to_2_seconds": any(
+            segment.get("type") == "silence" and 1000 <= segment.get("ms", 0) <= 2000
+            for segment in c_segments
+        ),
+        "thinking_native_filler_exercised": any("(uncertain)" in segment.get("fish_text", "") for segment in c_segments),
+    }
+    quality_study = None
+    quality_study_valid = False
+    quality_study_error = None
+    if QUALITY_STUDY.is_file():
+        try:
+            quality_study = json.loads(QUALITY_STUDY.read_text(encoding="utf-8"))
+            validate_study(quality_study)
+            quality_study_valid = True
+        except (json.JSONDecodeError, OSError, ValueError) as exc:
+            quality_study_error = str(exc)
+    artifact = {
+        "schema_version": 3,
+        "experiment": "6-6",
+        "timestamp_utc": datetime.now(timezone.utc).isoformat(),
+        "artifact_generation": {
+            "recorded_timestamp_utc": run["timestamp_utc"],
+            "provider": run["provider"],
+            "backend": run["backend"],
+            "source_reference_id_sha256": hashlib.sha256(manifest["source_reference_id"].encode()).hexdigest(),
+            "source_reference_value_saved_in_manifest": True,
+            "estimated_paid_api_requests": 30,
+            "request_count_basis": "24 reference renders + A(1) + B(1) + C(4 speech segments); local silence is not an API call",
+            "provider_reported_cost_usd": None,
+            "cost_note": "Fish SDK responses did not expose monetary charges; consult the provider billing ledger.",
+        },
+        "validation_provenance": {
+            "platform": platform.platform(),
+            "python": platform.python_version(),
+            "manifest_sha256": sha256(MANIFEST),
+            "run_evidence_sha256": sha256(RUN),
+            "implementation_sha256": {
+                name: sha256(HERE / name) for name in (
+                    "demo.py", "tts.py", "markup.py", "voice_library.py", "evaluate_audio_quality.py"
+                )
+            },
+        },
+        "reference_statistics": {
+            "count": len(reference_checks),
+            "minimum_duration_seconds": min(item["duration_seconds"] for item in reference_checks),
+            "maximum_duration_seconds": max(item["duration_seconds"] for item in reference_checks),
+            "mean_duration_seconds": sum(item["duration_seconds"] for item in reference_checks) / len(reference_checks),
+        },
+        "reference_checks": reference_checks,
+        "outputs": outputs,
+        "qualitative_study": {
+            "path": str(QUALITY_STUDY.relative_to(HERE)),
+            "present": QUALITY_STUDY.is_file(),
+            "valid": quality_study_valid,
+            "error": quality_study_error,
+            "judge_type": (quality_study or {}).get("study_design", {}).get("judge_type"),
+            "provider": (quality_study or {}).get("provider"),
+            "model": (quality_study or {}).get("model"),
+            "aggregate": (quality_study or {}).get("aggregate"),
+            "sha256": sha256(QUALITY_STUDY) if QUALITY_STUDY.is_file() else None,
+        },
+        "acceptance": {
+            "structural_and_media_gates": gates,
+            "structural_and_media_passed": all(gates.values()),
+            "qualitative_listening_study_present": quality_study_valid,
+            "qualitative_judge_is_human_mos": False,
+            "near_human_customer_service_claim_evaluated": quality_study_valid,
+            "manuscript_quality_claim_reproduced": (
+                quality_study.get("aggregate", {}).get("manuscript_quality_claim_reproduced")
+                if quality_study_valid else None
+            ),
+            "experiment_execution_complete": all(gates.values()) and quality_study_valid,
+            "statement": (
+                "Real Fish S1 media and a schema-checked, position-balanced multimodal listening study "
+                "complete the A/B/C experiment. The saved result reports independently whether the "
+                "manuscript's subjective ordering reproduced; this is not a human MOS panel."
+                if quality_study_valid else
+                "Real Fish S1 media fulfills construction, but the blinded qualitative study is absent or invalid."
+            ),
+        },
+    }
+    output = HERE / "validation" / "acceptance.json"
+    output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(output)
+    return 0 if artifact["acceptance"]["experiment_execution_complete"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())