ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter8/MiniMind-pretrain/validation/validate_evidence.py

@@ -0,0 +1,313 @@
+#!/usr/bin/env python3
+"""Fail-closed validator for Experiment 8-3 retained training evidence."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+
+HERE = Path(__file__).resolve().parent
+EXPERIMENT_DIR = HERE.parent
+REPO_ROOT = EXPERIMENT_DIR.parents[1]
+LATEST_PATH = HERE / "latest.json"
+
+ARMS = {"original", "qk_norm_muon"}
+STAGES = {"pretrain", "sft", "dpo"}
+EXPECTED_COUNTS = {
+    ("original", "pretrain"): 7,
+    ("original", "sft"): 8,
+    ("original", "dpo"): 9,
+    ("qk_norm_muon", "pretrain"): 7,
+    ("qk_norm_muon", "sft"): 9,
+    ("qk_norm_muon", "dpo"): 9,
+}
+EXPECTED_SOURCE_REVISION = "8bdc5d97d5845a8c1ac2ed56a5b8b4c0d0fb0795"
+EXPECTED_DATASET_REVISION = "84983ed4dec7836d240577760c1d6be5d4cabcf9"
+EXPECTED_DATASET_FILES = {
+    "pretrain_hq.jsonl": (
+        "9801b0d2210c61c2e4bc130f6dc4b3c870698a88d04af8f103c23dd5f0ce2440",
+        1_669_750_047,
+    ),
+    "sft_512.jsonl": (
+        "053b7d09574e48a86232e929211434ff9e5016c6ed13312e63687dd52edcbebf",
+        7_531_517_862,
+    ),
+    "dpo.jsonl": (
+        "ee934a8a455ccc99d1334d63e1254dd1d64f497fd067cfcbb71e3043f5b46768",
+        53_653_322,
+    ),
+}
+SECRET_PATTERNS = (
+    re.compile(r"(?i)authorization\s*[:=]\s*bearer\s+\S+"),
+    re.compile(r"(?i)(?:api[_-]?key|secret)\s*[:=]\s*[A-Za-z0-9._-]{16,}"),
+    re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b"),
+)
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def load_json(path: Path) -> dict[str, Any]:
+    value = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(value, dict):
+        raise AssertionError(f"{path} must contain a JSON object")
+    return value
+
+
+def resolve_relative(base: Path, relative: str) -> Path:
+    path = (base / relative).resolve()
+    if not path.is_relative_to(base.resolve()):
+        raise AssertionError(f"path escapes evidence root: {relative}")
+    return path
+
+
+def check_record(path: Path, record: dict[str, Any]) -> None:
+    if not path.is_file():
+        raise AssertionError(f"missing declared file: {path}")
+    if path.stat().st_size != record.get("bytes"):
+        raise AssertionError(f"byte count mismatch: {path}")
+    if sha256_file(path) != record.get("sha256"):
+        raise AssertionError(f"SHA-256 mismatch: {path}")
+
+
+def parse_response_content(content: str) -> dict[str, Any]:
+    stripped = content.strip()
+    if stripped.startswith("```"):
+        stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
+        stripped = re.sub(r"\s*```$", "", stripped)
+    value = json.loads(stripped)
+    if not isinstance(value, dict):
+        raise AssertionError("judge response content must decode to an object")
+    return value
+
+
+def find_pair(
+    retained: dict[str, Any], arm: str, stage: str, keyword: str
+) -> dict[str, str]:
+    cells = [
+        cell
+        for cell in retained["cells"]
+        if cell.get("arm") == arm and cell.get("stage") == stage
+    ]
+    if len(cells) != 1:
+        raise AssertionError(f"missing or duplicate retained cell: {arm}/{stage}")
+    matches = [
+        pair
+        for pair in cells[0]["pairs"]
+        if keyword.lower() in pair.get("prompt", "").lower()
+    ]
+    if len(matches) != 1:
+        raise AssertionError(f"missing or duplicate selected prompt: {arm}/{stage}/{keyword}")
+    return matches[0]
+
+
+def validate(latest_path: Path = LATEST_PATH) -> dict[str, Any]:
+    latest = load_json(latest_path)
+    if latest.get("experiment") != "8-3" or latest.get("status") != "passed":
+        raise AssertionError("latest pointer is not a passed Experiment 8-3 run")
+    run_dir = resolve_relative(EXPERIMENT_DIR, latest["run_dir"])
+    manifest_path = run_dir / "manifest.json"
+    if sha256_file(manifest_path) != latest.get("manifest_sha256"):
+        raise AssertionError("latest manifest hash mismatch")
+    manifest = load_json(manifest_path)
+    if manifest.get("run_id") != latest.get("run_id"):
+        raise AssertionError("run ID mismatch between latest and manifest")
+    if manifest.get("experiment") != "8-3" or manifest.get("status") != "passed":
+        raise AssertionError("manifest is not a passed Experiment 8-3 run")
+    if manifest.get("checkpoint_policy") != "not distributed; not an acceptance artifact":
+        raise AssertionError("manifest checkpoint policy is incorrect")
+
+    inputs = manifest.get("inputs")
+    artifacts = manifest.get("artifacts")
+    if not isinstance(inputs, list) or len(inputs) != 5:
+        raise AssertionError("manifest must bind exactly five repository inputs")
+    if not isinstance(artifacts, list) or len(artifacts) != 5:
+        raise AssertionError("manifest must bind exactly five run artifacts")
+    for record in inputs:
+        check_record(resolve_relative(REPO_ROOT, record["path"]), record)
+    for record in artifacts:
+        check_record(resolve_relative(run_dir, record["path"]), record)
+
+    retained = load_json(run_dir / "retained_outputs.json")
+    report_record = next(
+        (record for record in inputs if record.get("path") == retained.get("source_report")),
+        None,
+    )
+    if report_record is None or retained.get("source_report_sha256") != report_record.get("sha256"):
+        raise AssertionError("retained source-report hash does not match the manifest input")
+    if retained.get("cell_count") != 6 or retained.get("output_count") != 49:
+        raise AssertionError("retained report must contain six cells and 49 outputs")
+    if set(retained.get("arms", [])) != ARMS or set(retained.get("stages", [])) != STAGES:
+        raise AssertionError("retained report arm/stage coverage is incomplete")
+    cells = retained.get("cells")
+    if not isinstance(cells, list) or len(cells) != 6:
+        raise AssertionError("retained cells are malformed")
+    combos: set[tuple[str, str]] = set()
+    for cell in cells:
+        combo = (cell.get("arm"), cell.get("stage"))
+        if combo in combos or combo not in EXPECTED_COUNTS:
+            raise AssertionError(f"duplicate or unexpected cell: {combo}")
+        combos.add(combo)
+        pairs = cell.get("pairs")
+        if not isinstance(pairs, list) or len(pairs) != EXPECTED_COUNTS[combo]:
+            raise AssertionError(f"wrong retained pair count for {combo}")
+        if any(not pair.get("prompt") or not pair.get("output") for pair in pairs):
+            raise AssertionError(f"empty retained prompt/output in {combo}")
+    if combos != set(EXPECTED_COUNTS):
+        raise AssertionError("not all arm/stage cells are present")
+
+    receipts_root = load_json(run_dir / "judge_receipts.json")
+    if receipts_root.get("credential_headers_retained") is not False:
+        raise AssertionError("credential header retention must be explicitly false")
+    calls = receipts_root.get("calls")
+    if not isinstance(calls, list) or len(calls) != 8:
+        raise AssertionError("exactly eight raw judge calls are required")
+    response_ids: set[str] = set()
+    normalized_rows: dict[str, Any] = {}
+    for expected_case_id, call in enumerate(calls, start=1):
+        if call.get("case_id") != expected_case_id or call.get("http_status") != 200:
+            raise AssertionError("judge calls must be successful and ordered by case ID")
+        if call.get("credential_headers_retained") is not False:
+            raise AssertionError("per-call credential retention boundary is missing")
+        response_id = call.get("response_id")
+        if not isinstance(response_id, str) or not response_id or response_id in response_ids:
+            raise AssertionError("judge response IDs must be present and unique")
+        response_ids.add(response_id)
+        if call.get("latency_ms", 0) <= 0 or call.get("usage", {}).get("total_tokens", 0) <= 0:
+            raise AssertionError("judge usage and positive latency must be retained")
+        raw_response = call.get("response", {})
+        if raw_response.get("id") != response_id or raw_response.get("usage") != call.get("usage"):
+            raise AssertionError("copied response ID/usage does not match raw response")
+        try:
+            content = raw_response["choices"][0]["message"]["content"]
+        except (KeyError, IndexError, TypeError) as exc:
+            raise AssertionError("raw judge response is missing message content") from exc
+        judgment = parse_response_content(content)
+        if judgment != call.get("judgment"):
+            raise AssertionError("normalized judgment does not match raw response content")
+        if str(judgment.get("case_id")) != str(expected_case_id):
+            raise AssertionError("raw judgment has the wrong case ID")
+        if set(judgment.get("candidates", {})) != {"A", "B"}:
+            raise AssertionError("judge judgment must score A and B")
+        if judgment.get("winner") not in {"A", "B", "tie"}:
+            raise AssertionError("judge winner is invalid")
+
+        mapping = call.get("blind_map")
+        if not isinstance(mapping, dict) or set(mapping) != {"A", "B"} or set(mapping.values()) != ARMS:
+            raise AssertionError("blind mapping must cover both arms")
+        request = call.get("request")
+        request_text = json.dumps(request, ensure_ascii=False).lower()
+        if "qk_norm_muon" in request_text or '"original"' in request_text:
+            raise AssertionError("judge request leaks an arm identity")
+        try:
+            user_payload = json.loads(request["messages"][1]["content"])
+            request_candidates = user_payload["candidates"]
+        except (KeyError, IndexError, TypeError, json.JSONDecodeError) as exc:
+            raise AssertionError("judge request is missing structured candidates") from exc
+        if user_payload.get("case_id") != expected_case_id:
+            raise AssertionError("judge request case ID mismatch")
+        for label, arm in mapping.items():
+            retained_pair = find_pair(retained, arm, call["stage"], call["keyword"])
+            expected_candidate = {
+                "historical_prompt": retained_pair["prompt"],
+                "historical_output": retained_pair["output"],
+            }
+            if request_candidates.get(label) != expected_candidate:
+                raise AssertionError("raw judge request is not bound to the retained output")
+        normalized_rows[str(expected_case_id)] = {
+            mapping[label]: score for label, score in judgment["candidates"].items()
+        }
+
+    contract = load_json(run_dir / "reproduction_contract.json")
+    future = contract.get("future_reproduction", {})
+    source = future.get("source", {})
+    dataset = future.get("dataset", {})
+    if source.get("revision") != EXPECTED_SOURCE_REVISION:
+        raise AssertionError("frozen MiniMind source revision mismatch")
+    source_hashes = source.get("file_sha256")
+    if not isinstance(source_hashes, dict) or len(source_hashes) < 12:
+        raise AssertionError("frozen source file hashes are incomplete")
+    if any(not re.fullmatch(r"[0-9a-f]{64}", value) for value in source_hashes.values()):
+        raise AssertionError("invalid frozen source SHA-256")
+    if dataset.get("revision") != EXPECTED_DATASET_REVISION:
+        raise AssertionError("frozen dataset revision mismatch")
+    dataset_files = dataset.get("files", {})
+    for name, (expected_hash, expected_bytes) in EXPECTED_DATASET_FILES.items():
+        record = dataset_files.get(name, {})
+        if record.get("lfs_sha256") != expected_hash or record.get("bytes") != expected_bytes:
+            raise AssertionError(f"frozen dataset file mismatch: {name}")
+    if len(future.get("commands", {})) != 6:
+        raise AssertionError("all six reproduction commands are required")
+    boundary = contract.get("historical_evidence_boundary", {})
+    for key in (
+        "historical_source_revision_retained",
+        "historical_dataset_hashes_retained",
+        "historical_checkpoint_hashes_retained",
+        "historical_stepwise_training_logs_retained",
+    ):
+        if boundary.get(key) is not False:
+            raise AssertionError(f"historical provenance boundary is not explicit: {key}")
+    mechanisms = contract.get("model_and_training", {}).get("source_verified_mechanisms", {})
+    if not mechanisms or not all(value is True for value in mechanisms.values()):
+        raise AssertionError("source mechanism assertions are incomplete")
+    policy = contract.get("checkpoint_policy", {})
+    if policy.get("distributed_with_book") is not False or policy.get("acceptance_artifact") is not False:
+        raise AssertionError("checkpoint policy does not match the book contract")
+
+    summary = load_json(run_dir / "summary.json")
+    if summary.get("status") != "passed" or summary.get("acceptance", {}).get("passed") is not True:
+        raise AssertionError("summary acceptance did not pass")
+    acceptance = summary["acceptance"]
+    if not all(value is True for key, value in acceptance.items() if key != "passed"):
+        failed = [key for key, value in acceptance.items() if key != "passed" and value is not True]
+        raise AssertionError(f"required acceptance gates failed: {failed}")
+    if summary.get("per_case_arm_scores") != normalized_rows:
+        raise AssertionError("summary scores do not match raw judge responses")
+    findings = summary.get("scientific_findings", {})
+    if not isinstance(findings.get("blind_judge_prefers_qk_norm_muon_overall"), bool):
+        raise AssertionError("comparative scientific finding is missing")
+    if findings.get("reported_loss_comparison_retained_but_not_independently_recomputed") is not True:
+        raise AssertionError("loss-evidence qualification is missing")
+
+    for record in artifacts:
+        path = resolve_relative(run_dir, record["path"])
+        if path.suffix not in {".json", ".md"}:
+            continue
+        text = path.read_text(encoding="utf-8")
+        for pattern in SECRET_PATTERNS:
+            if pattern.search(text):
+                raise AssertionError(f"possible credential in retained artifact: {path.name}")
+
+    return {
+        "experiment": "8-3",
+        "run_id": latest["run_id"],
+        "status": "passed",
+        "inputs_verified": len(inputs),
+        "artifacts_verified": len(artifacts),
+        "judge_receipts_verified": len(calls),
+        "outputs_verified": retained["output_count"],
+        "manifest_sha256": latest["manifest_sha256"],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--latest", type=Path, default=LATEST_PATH)
+    args = parser.parse_args()
+    result = validate(args.latest.resolve())
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())