ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter8/MiniMind-pretrain/validation/run_training_report_audit.py

@@ -0,0 +1,760 @@
+#!/usr/bin/env python3
+"""Build checkpoint-free retained training evidence for Experiment 8-3.
+
+The book already contains the author's historical six-cell evaluation report:
+original versus QK-Norm + Muon at pretrain, SFT, and DPO.  This program does
+not pretend to rerun the GPU training job.  It extracts every saved generation,
+submits a preregistered stage-balanced subset to an arm-blind external judge,
+and binds the raw report, judge receipts, immutable future-reproduction source
+and dataset revisions, environment lock, findings, and limitations into a
+content-hashed evidence package.  Checkpoints are intentionally not published
+and are not an acceptance artifact for book training experiments.
+"""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import hashlib
+import json
+import os
+import random
+import re
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+HERE = Path(__file__).resolve().parent
+EXPERIMENT_DIR = HERE.parent
+REPO_ROOT = EXPERIMENT_DIR.parents[1]
+REPORT_PATH = EXPERIMENT_DIR / "README.md"
+RUNS_DIR = HERE / "runs"
+LATEST_PATH = HERE / "latest.json"
+
+DEFAULT_RUN_ID = "exp8-3-training-report-20260731-v1"
+DEFAULT_ENDPOINT = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
+DEFAULT_MODEL = "doubao-seed-1-6-250615"
+BLIND_SEED = 730731
+
+SOURCE_REVISION = "8bdc5d97d5845a8c1ac2ed56a5b8b4c0d0fb0795"
+DATASET_REVISION = "84983ed4dec7836d240577760c1d6be5d4cabcf9"
+SOURCE_FILES = {
+    "model/model_minimind.py": "2d33988711c704be6a22c4c61489b23106a2340a7cb8b97ebe3e40f30819cbb0",
+    "model/tokenizer.json": "e489029175fb3f94b8211a120a72a2ee41a664db65b828d077c7bde989c845a9",
+    "model/tokenizer_config.json": "190cc4738bac3b6f6b563376019c581b320fdb0260a03b9d5ab806296c8c6bb8",
+    "dataset/lm_dataset.py": "213726b1781289548784220b2b2db48fe97d84f3b67ac0cd70186cbbcb7b5d2c",
+    "trainer/muon.py": "00c2c6a225edeb55433df0724c3c74f6ff98ac4b2cc73c4aafcff686824f6267",
+    "trainer/train_pretrain.py": "ddd122645a9f1043bc8dac69a81ac51d2df95df8745d25faed7963d38fedc328",
+    "trainer/train_pretrain_muon.py": "fc83d07754ec3a8c156b6b8bfc0fd4326edecb72efabc5e08ae4ff5e3a7029bc",
+    "trainer/train_full_sft.py": "a57422f1df80bf2867f31f3b4a646a92ac7f66729e98a3b32cd1ec4d6780cb8b",
+    "trainer/train_full_sft_muon.py": "acd0b7db5b1d8b25d3c3103f92d68a7d381f9322a1b33bbec34be3d005930bad",
+    "trainer/train_dpo.py": "97f2c31cc8bc21a777e2efcb5e2fa35a49e4e9e3698db120148f8a0b2f678449",
+    "eval_model.py": "43930a4b55048a4a3ffa17eb78ae67d59582d639aa9365f0bbf41ba149128af8",
+    "requirements.txt": "23f4cea09281765eec7cf03e28231425638e8e42580f418781fed166b75af968",
+}
+DATASET_FILES = {
+    "pretrain_hq.jsonl": {
+        "lfs_sha256": "9801b0d2210c61c2e4bc130f6dc4b3c870698a88d04af8f103c23dd5f0ce2440",
+        "bytes": 1_669_750_047,
+    },
+    "sft_512.jsonl": {
+        "lfs_sha256": "053b7d09574e48a86232e929211434ff9e5016c6ed13312e63687dd52edcbebf",
+        "bytes": 7_531_517_862,
+    },
+    "dpo.jsonl": {
+        "lfs_sha256": "ee934a8a455ccc99d1334d63e1254dd1d64f497fd067cfcbb71e3043f5b46768",
+        "bytes": 53_653_322,
+    },
+}
+
+ARMS = ("original", "qk_norm_muon")
+STAGES = ("pretrain", "sft", "dpo")
+EXPECTED_COUNTS = {
+    ("original", "pretrain"): 7,
+    ("original", "sft"): 8,
+    ("original", "dpo"): 9,
+    ("qk_norm_muon", "pretrain"): 7,
+    ("qk_norm_muon", "sft"): 9,
+    ("qk_norm_muon", "dpo"): 9,
+}
+SELECTED_CASES = (
+    {"case_id": 1, "stage": "pretrain", "keyword": "highest mountain", "task": "Continue the prompt by identifying the highest mountain in the world accurately."},
+    {"case_id": 2, "stage": "pretrain", "keyword": "carbon dioxide", "task": "Continue the prompt with an accurate statement about carbon dioxide in air."},
+    {"case_id": 3, "stage": "sft", "keyword": "speed of light", "task": "Explain the physical concept of the speed of light in detail."},
+    {"case_id": 4, "stage": "sft", "keyword": "how to understand chatgpt", "task": "Explain what ChatGPT is and how it works."},
+    {"case_id": 5, "stage": "sft", "keyword": "history of the united states", "task": "Introduce the history of the United States."},
+    {"case_id": 6, "stage": "dpo", "keyword": "speed of light", "task": "Explain the physical concept of the speed of light in detail."},
+    {"case_id": 7, "stage": "dpo", "keyword": "how to understand chatgpt", "task": "Explain what ChatGPT is and how it works."},
+    {"case_id": 8, "stage": "dpo", "keyword": "history of the united states", "task": "Introduce the history of the United States."},
+)
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def canonical_json_bytes(value: Any) -> bytes:
+    return (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode()
+
+
+def write_json(path: Path, value: Any) -> None:
+    path.write_bytes(canonical_json_bytes(value))
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _terminal_transcript(section: str) -> tuple[str, str]:
+    """Return the terminal header and prompt/output transcript from one cell.
+
+    The historical Markdown has one stray closing fence immediately before the
+    improved-SFT Lu Xun answer.  Treat fences as presentation markup rather
+    than semantic delimiters so that the retained answer is not silently lost.
+    """
+
+    prompt_position = section.find("👶:")
+    if prompt_position < 0:
+        raise ValueError("model section has no user prompt marker")
+    header = section[:prompt_position].replace("```", "").strip()
+    transcript = section[prompt_position:]
+    analysis = re.search(r"(?m)^\*\*[^\n]*Analysis[^\n]*\*\*:?\s*$", transcript)
+    if analysis:
+        transcript = transcript[: analysis.start()]
+    transcript = re.sub(r"(?m)^```\s*$", "", transcript).strip()
+    return header, transcript
+
+
+def _parse_pairs(transcript: str) -> list[dict[str, str]]:
+    pattern = re.compile(
+        r"^👶:\s*(.*?)\n🤖️:\s*(.*?)(?=\n(?:\s*\n)*👶:|\Z)",
+        flags=re.DOTALL | re.MULTILINE,
+    )
+    pairs = []
+    for match in pattern.finditer(transcript):
+        prompt = match.group(1).strip()
+        output = match.group(2).strip()
+        if not prompt or not output:
+            raise ValueError("empty prompt or output in historical transcript")
+        pairs.append({"prompt": prompt, "output": output})
+    return pairs
+
+
+def parse_retained_outputs(report_path: Path = REPORT_PATH) -> dict[str, Any]:
+    """Extract all six historical LLM evaluation cells from the bilingual report."""
+
+    text = report_path.read_text(encoding="utf-8")
+    start = text.index("## Language Model Training Results Analysis")
+    end = text.index("# Analysis of Vision-Language Model Training Results", start)
+    llm_text = text[start:end]
+    arm_markers = {
+        "original": "## Without Muon Optimizer (Original Architecture)",
+        "qk_norm_muon": "## With Muon Optimizer and QK Norm (Improved Architecture)",
+    }
+    cells: list[dict[str, Any]] = []
+    for arm_index, arm in enumerate(ARMS):
+        arm_start = llm_text.index(arm_markers[arm])
+        arm_end = (
+            llm_text.index(arm_markers[ARMS[arm_index + 1]], arm_start)
+            if arm_index + 1 < len(ARMS)
+            else len(llm_text)
+        )
+        arm_text = llm_text[arm_start:arm_end]
+        for stage_index, stage in enumerate(STAGES):
+            heading = {"pretrain": "### Pretrain Model", "sft": "### SFT Model", "dpo": "### DPO Model"}[stage]
+            cell_start = arm_text.index(heading)
+            next_positions = [
+                arm_text.find(next_heading, cell_start + len(heading))
+                for next_heading in ("### Pretrain Model", "### SFT Model", "### DPO Model")
+            ]
+            next_positions = [position for position in next_positions if position >= 0]
+            cell_end = min(next_positions) if next_positions else len(arm_text)
+            header, transcript = _terminal_transcript(arm_text[cell_start:cell_end])
+            pairs = _parse_pairs(transcript)
+            expected = EXPECTED_COUNTS[(arm, stage)]
+            if len(pairs) != expected:
+                raise ValueError(f"{arm}/{stage}: expected {expected} pairs, found {len(pairs)}")
+            cells.append(
+                {
+                    "arm": arm,
+                    "stage": stage,
+                    "terminal_header": header,
+                    "pair_count": len(pairs),
+                    "pairs": pairs,
+                }
+            )
+
+    return {
+        "schema_version": "exp8-3-retained-outputs-v1",
+        "experiment": "8-3",
+        "source_report": str(report_path.relative_to(REPO_ROOT)),
+        "source_report_sha256": sha256_file(report_path),
+        "arms": list(ARMS),
+        "stages": list(STAGES),
+        "cell_count": len(cells),
+        "output_count": sum(cell["pair_count"] for cell in cells),
+        "cells": cells,
+    }
+
+
+def _find_pair(retained: dict[str, Any], arm: str, stage: str, keyword: str) -> dict[str, str]:
+    cell = next(cell for cell in retained["cells"] if cell["arm"] == arm and cell["stage"] == stage)
+    matches = [pair for pair in cell["pairs"] if keyword in pair["prompt"].lower()]
+    if len(matches) != 1:
+        raise ValueError(f"{arm}/{stage}/{keyword}: expected one prompt, found {len(matches)}")
+    return matches[0]
+
+
+def selected_comparisons(retained: dict[str, Any]) -> list[dict[str, Any]]:
+    comparisons = []
+    for case in SELECTED_CASES:
+        rows = {
+            arm: _find_pair(retained, arm, case["stage"], case["keyword"])
+            for arm in ARMS
+        }
+        comparisons.append({**case, "arms": rows})
+    return comparisons
+
+
+def blind_mapping(case_id: int) -> dict[str, str]:
+    arms = list(ARMS)
+    random.Random(BLIND_SEED + case_id).shuffle(arms)
+    return dict(zip(("A", "B"), arms, strict=True))
+
+
+def judge_payload(comparison: dict[str, Any], mapping: dict[str, str], model: str) -> dict[str, Any]:
+    candidates = {
+        label: {
+            "historical_prompt": comparison["arms"][arm]["prompt"],
+            "historical_output": comparison["arms"][arm]["output"],
+        }
+        for label, arm in mapping.items()
+    }
+    required = {
+        "case_id": comparison["case_id"],
+        "candidates": {
+            label: {
+                "language_fluency": "number 0-5",
+                "instruction_following": "number 0-5",
+                "factuality": "number 0-5",
+                "factual_errors": ["specific material errors; empty only if none"],
+                "rationale": "brief evidence-based explanation",
+            }
+            for label in ("A", "B")
+        },
+        "winner": "A, B, or tie",
+    }
+    return {
+        "model": model,
+        "temperature": 0,
+        "response_format": {"type": "json_object"},
+        "messages": [
+            {
+                "role": "system",
+                "content": (
+                    "You are an independent evaluator of two anonymous small-language-model outputs. "
+                    "Do not infer model identity, architecture, optimizer, or training stage. Score only "
+                    "the supplied text. Penalize hallucinations, unsafe medical specificity, repetition, "
+                    "and non-answers. Return one JSON object with exactly the requested fields."
+                ),
+            },
+            {
+                "role": "user",
+                "content": json.dumps(
+                    {
+                        "case_id": comparison["case_id"],
+                        "task": comparison["task"],
+                        "rubric": {
+                            "language_fluency": "0 unreadable; 3 understandable with defects; 5 coherent and natural",
+                            "instruction_following": "0 non-answer; 3 partial; 5 directly and fully answers",
+                            "factuality": "0 dominated by falsehoods; 3 mixed/minor errors; 5 no material error",
+                        },
+                        "candidates": candidates,
+                        "required_json_shape": required,
+                    },
+                    ensure_ascii=False,
+                    sort_keys=True,
+                ),
+            },
+        ],
+    }
+
+
+def extract_json_object(content: str) -> dict[str, Any]:
+    stripped = content.strip()
+    if stripped.startswith("```"):
+        stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
+        stripped = re.sub(r"\s*```$", "", stripped)
+    parsed = json.loads(stripped)
+    if not isinstance(parsed, dict):
+        raise ValueError("judge content must decode to an object")
+    return parsed
+
+
+def validate_judgment(judgment: dict[str, Any], case_id: int) -> None:
+    if str(judgment.get("case_id")) != str(case_id):
+        raise ValueError(
+            f"judge returned the wrong case_id: expected {case_id}, got {judgment.get('case_id')!r}"
+        )
+    candidates = judgment.get("candidates")
+    if not isinstance(candidates, dict) or set(candidates) != {"A", "B"}:
+        raise ValueError("judge must score A and B exactly")
+    for label in ("A", "B"):
+        row = candidates[label]
+        if not isinstance(row, dict):
+            raise ValueError(f"candidate {label} score must be an object")
+        for metric in ("language_fluency", "instruction_following", "factuality"):
+            score = row.get(metric)
+            if not isinstance(score, (int, float)) or isinstance(score, bool) or not 0 <= score <= 5:
+                raise ValueError(f"candidate {label} has invalid {metric}: {score!r}")
+        if not isinstance(row.get("factual_errors"), list):
+            raise ValueError(f"candidate {label} factual_errors must be a list")
+        if not isinstance(row.get("rationale"), str) or not row["rationale"].strip():
+            raise ValueError(f"candidate {label} rationale is missing")
+    if judgment.get("winner") not in {"A", "B", "tie"}:
+        raise ValueError("judge winner must be A, B, or tie")
+
+
+def call_judge(
+    comparison: dict[str, Any], *, endpoint: str, model: str, api_key: str, timeout: float
+) -> dict[str, Any]:
+    mapping = blind_mapping(comparison["case_id"])
+    payload = judge_payload(comparison, mapping, model)
+    request = urllib.request.Request(
+        endpoint,
+        data=json.dumps(payload, ensure_ascii=False).encode(),
+        method="POST",
+        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+    )
+    started = time.perf_counter()
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            raw_body = response.read()
+            http_status = response.status
+    except urllib.error.HTTPError as exc:
+        body = exc.read().decode(errors="replace")
+        raise RuntimeError(f"judge HTTP {exc.code}: {body[:500]}") from exc
+    latency_ms = round((time.perf_counter() - started) * 1000, 3)
+    raw_response = json.loads(raw_body)
+    content = raw_response["choices"][0]["message"]["content"]
+    judgment = extract_json_object(content)
+    validate_judgment(judgment, comparison["case_id"])
+    response_id = raw_response.get("id")
+    usage = raw_response.get("usage")
+    if not isinstance(response_id, str) or not response_id:
+        raise ValueError("judge response has no response ID")
+    if not isinstance(usage, dict) or not isinstance(usage.get("total_tokens"), int):
+        raise ValueError("judge response has no complete usage object")
+    return {
+        "case_id": comparison["case_id"],
+        "stage": comparison["stage"],
+        "keyword": comparison["keyword"],
+        "task": comparison["task"],
+        "provider": "ark",
+        "endpoint": endpoint,
+        "credential_env": "ARK_API_KEY",
+        "credential_headers_retained": False,
+        "blind_seed": BLIND_SEED,
+        "blind_map": mapping,
+        "request": payload,
+        "http_status": http_status,
+        "response": raw_response,
+        "response_id": response_id,
+        "usage": usage,
+        "latency_ms": latency_ms,
+        "judgment": judgment,
+    }
+
+
+def reproduction_contract() -> dict[str, Any]:
+    return {
+        "schema_version": "exp8-3-reproduction-contract-v1",
+        "experiment": "8-3",
+        "historical_evidence_boundary": {
+            "historical_training_executed": True,
+            "six_historical_evaluation_transcripts_retained": True,
+            "historical_source_revision_retained": False,
+            "historical_dataset_hashes_retained": False,
+            "historical_checkpoint_hashes_retained": False,
+            "historical_stepwise_training_logs_retained": False,
+            "claim": (
+                "The author's retained report supports that original and QK-Norm+Muon 104.03M models "
+                "were evaluated after pretrain, SFT, and DPO. It does not establish byte identity of the "
+                "historical checkpoints, datasets, source checkout, or every loss point."
+            ),
+        },
+        "future_reproduction": {
+            "source": {
+                "repository": "bojieli/minimind",
+                "revision": SOURCE_REVISION,
+                "selected_at": "2026-07-31",
+                "not_claimed_as_historical_revision": True,
+                "file_sha256": SOURCE_FILES,
+            },
+            "dataset": {
+                "repository": "jingyaogong/minimind_dataset",
+                "revision": DATASET_REVISION,
+                "selected_at": "2026-07-31",
+                "not_claimed_as_historical_revision": True,
+                "files": DATASET_FILES,
+            },
+            "commands": {
+                "original_pretrain": "torchrun --nproc_per_node=8 trainer/train_pretrain.py --epochs 10 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/pretrain_hq.jsonl --use_wandb",
+                "improved_pretrain": "torchrun --nproc_per_node=8 trainer/train_pretrain_muon.py --epochs 10 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/pretrain_hq.jsonl --use_wandb",
+                "original_sft": "torchrun --nproc_per_node=8 trainer/train_full_sft.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/sft_512.jsonl --use_wandb",
+                "improved_sft": "torchrun --nproc_per_node=8 trainer/train_full_sft_muon.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/sft_512.jsonl --use_wandb",
+                "original_dpo": "torchrun --nproc_per_node=8 trainer/train_dpo.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --init_from out/full_sft_768.pth --data_path dataset/dpo.jsonl --use_wandb",
+                "improved_dpo": "torchrun --nproc_per_node=8 trainer/train_dpo.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --init_from out/full_sft_muon_768.pth --data_path dataset/dpo.jsonl --use_wandb",
+            },
+            "environment": {
+                "book_pyproject": "pyproject.toml",
+                "book_lock": "uv.lock",
+                "install": "uv sync --locked --python 3.12 --extra ch7 --extra dev",
+                "boundary": (
+                    "The book lock freezes a future software environment. The pinned upstream requirements "
+                    "file itself is unversioned, and the future GPU/CUDA stack has not been exercised here."
+                ),
+            },
+        },
+        "model_and_training": {
+            "reported_parameter_count_millions": 104.03,
+            "architecture": {"hidden_size": 768, "layers": 16, "sequence_length": 512},
+            "stages": list(STAGES),
+            "arms": list(ARMS),
+            "source_verified_mechanisms": {
+                "qk_norm_before_rope": True,
+                "muon_for_two_dimensional_non_embedding_weights": True,
+                "adamw_for_embeddings_norms_and_lm_head": True,
+                "dpo_uses_adamw_from_arm_specific_sft_checkpoint": True,
+            },
+            "reported_scalars_without_stepwise_logs": {
+                "steps_to_loss_3_original": 36,
+                "steps_to_loss_3_qk_norm_muon": 12,
+                "final_loss_original": 2.0,
+                "final_loss_qk_norm_muon": 1.7,
+                "eight_rtx_4090_pretrain_hours": 6,
+                "eight_rtx_4090_sft_hours": 8,
+            },
+        },
+        "checkpoint_policy": {
+            "distributed_with_book": False,
+            "acceptance_artifact": False,
+            "required_artifact": "reproducible evidence-backed training report",
+            "reason": "Training checkpoints are intentionally not distributed to readers.",
+        },
+    }
+
+
+def mean(values: list[float]) -> float:
+    return round(sum(values) / len(values), 4)
+
+
+def summarize(retained: dict[str, Any], receipts: list[dict[str, Any]], contract: dict[str, Any]) -> dict[str, Any]:
+    metrics = ("language_fluency", "instruction_following", "factuality")
+    rows: dict[int, dict[str, dict[str, Any]]] = {}
+    wins = {"original": 0, "qk_norm_muon": 0, "tie": 0}
+    for receipt in receipts:
+        reverse = receipt["blind_map"]
+        rows[receipt["case_id"]] = {
+            reverse[label]: score for label, score in receipt["judgment"]["candidates"].items()
+        }
+        winner = receipt["judgment"]["winner"]
+        wins["tie" if winner == "tie" else reverse[winner]] += 1
+
+    arm_averages = {}
+    stage_averages = {}
+    for arm in ARMS:
+        arm_scores = [rows[case["case_id"]][arm] for case in SELECTED_CASES]
+        arm_averages[arm] = {
+            metric: mean([float(row[metric]) for row in arm_scores]) for metric in metrics
+        }
+        arm_averages[arm]["overall"] = mean(
+            [float(row[metric]) for row in arm_scores for metric in metrics]
+        )
+        stage_averages[arm] = {}
+        for stage in STAGES:
+            stage_scores = [
+                rows[case["case_id"]][arm] for case in SELECTED_CASES if case["stage"] == stage
+            ]
+            stage_averages[arm][stage] = {
+                metric: mean([float(row[metric]) for row in stage_scores]) for metric in metrics
+            }
+            stage_averages[arm][stage]["overall"] = mean(
+                [float(row[metric]) for row in stage_scores for metric in metrics]
+            )
+
+    findings = {
+        "blind_judge_overall_delta_qk_norm_muon_minus_original": round(
+            arm_averages["qk_norm_muon"]["overall"] - arm_averages["original"]["overall"], 4
+        ),
+        "blind_judge_prefers_qk_norm_muon_overall": (
+            arm_averages["qk_norm_muon"]["overall"] > arm_averages["original"]["overall"]
+        ),
+        "wins": wins,
+        "reported_loss_comparison_retained_but_not_independently_recomputed": True,
+    }
+    acceptance = {
+        "raw_historical_report_hashed": bool(retained["source_report_sha256"]),
+        "all_six_arm_stage_cells_retained": retained["cell_count"] == 6,
+        "all_expected_outputs_retained": retained["output_count"] == sum(EXPECTED_COUNTS.values()),
+        "pretrain_sft_and_dpo_compared": set(retained["stages"]) == set(STAGES),
+        "original_and_qk_norm_muon_compared": set(retained["arms"]) == set(ARMS),
+        "eight_stage_balanced_blind_judgments": len(receipts) == len(SELECTED_CASES),
+        "raw_judge_requests_responses_ids_usage_latency_retained": all(
+            receipt["response_id"]
+            and receipt["usage"].get("total_tokens", 0) > 0
+            and receipt["latency_ms"] > 0
+            for receipt in receipts
+        ),
+        "immutable_source_revision_and_file_hashes_frozen": bool(SOURCE_REVISION and SOURCE_FILES),
+        "immutable_dataset_revision_lfs_hashes_and_sizes_frozen": bool(DATASET_REVISION and DATASET_FILES),
+        "future_reproduction_commands_declared": len(contract["future_reproduction"]["commands"]) == 6,
+        "historical_provenance_limitations_explicit": (
+            contract["historical_evidence_boundary"]["historical_checkpoint_hashes_retained"] is False
+            and contract["historical_evidence_boundary"]["historical_stepwise_training_logs_retained"] is False
+        ),
+        "reported_loss_claims_qualified": findings[
+            "reported_loss_comparison_retained_but_not_independently_recomputed"
+        ],
+        "checkpoints_not_an_acceptance_artifact": (
+            contract["checkpoint_policy"]["acceptance_artifact"] is False
+        ),
+    }
+    passed = all(acceptance.values())
+    return {
+        "schema_version": "exp8-3-summary-v1",
+        "experiment": "8-3",
+        "status": "passed" if passed else "failed",
+        "judge": {
+            "provider": "ark",
+            "model": receipts[0]["request"]["model"],
+            "calls": len(receipts),
+            "response_ids": [receipt["response_id"] for receipt in receipts],
+            "total_tokens": sum(receipt["usage"]["total_tokens"] for receipt in receipts),
+            "total_latency_ms": round(sum(receipt["latency_ms"] for receipt in receipts), 3),
+            "blind_seed": BLIND_SEED,
+        },
+        "retained": {
+            "cells": retained["cell_count"],
+            "outputs": retained["output_count"],
+            "selected_comparisons": len(receipts),
+        },
+        "arm_averages": arm_averages,
+        "stage_averages": stage_averages,
+        "per_case_arm_scores": rows,
+        "scientific_findings": findings,
+        "acceptance": {**acceptance, "passed": passed},
+        "limitations": [
+            "Historical checkpoints are intentionally not distributed and were not recreated in this audit.",
+            "The historical source revision, dataset byte identities, RNG state, and stepwise loss logs were not retained.",
+            "Frozen source/data revisions and the book lock define a future reproduction contract, not historical provenance.",
+            "The independent judge covers eight preregistered comparisons; all other retained outputs remain available for inspection.",
+            "The historical outputs are English translations in a bilingual report, so translation may affect the judge scores.",
+        ],
+    }
+
+
+def render_report(summary: dict[str, Any]) -> str:
+    averages = summary["arm_averages"]
+    findings = summary["scientific_findings"]
+    return "\n".join(
+        [
+            "# Experiment 8-3 retained-training-report audit",
+            "",
+            "## Result",
+            "",
+            f"Status: **{summary['status']}**. The historical report retains "
+            f"{summary['retained']['outputs']} outputs across the original and QK-Norm + Muon arms "
+            "after pretrain, SFT, and DPO. Eight preregistered arm-blind comparisons were judged "
+            "from raw retained text by an independent ARK model.",
+            "",
+            "| Arm | Fluency | Instruction | Factuality | Overall |",
+            "| --- | ---: | ---: | ---: | ---: |",
+            f"| Original | {averages['original']['language_fluency']:.4f} | "
+            f"{averages['original']['instruction_following']:.4f} | "
+            f"{averages['original']['factuality']:.4f} | {averages['original']['overall']:.4f} |",
+            f"| QK-Norm + Muon | {averages['qk_norm_muon']['language_fluency']:.4f} | "
+            f"{averages['qk_norm_muon']['instruction_following']:.4f} | "
+            f"{averages['qk_norm_muon']['factuality']:.4f} | {averages['qk_norm_muon']['overall']:.4f} |",
+            "",
+            f"Observed blind-judge overall delta: **{findings['blind_judge_overall_delta_qk_norm_muon_minus_original']:+.4f}**. "
+            f"Pairwise decisions: {findings['wins']}.",
+            "",
+            "The report's loss claims (3.0 reached at 36 versus 12 reported steps; final loss 2.0 "
+            "versus 1.7) are retained as author-reported observations, not independently recomputed "
+            "measurements, because the historical stepwise logs were not preserved.",
+            "",
+            "## Provenance and reproduction boundary",
+            "",
+            "`reproduction_contract.json` freezes the MiniMind source revision, hashes the relevant "
+            "source files, freezes a dataset revision with the three Git-LFS object hashes and sizes, "
+            "and records all six future reproduction commands. These pins were selected for future "
+            "reproduction and are not represented as the exact historical checkout.",
+            "",
+            "Training checkpoints remain local by book policy and are not an acceptance artifact. "
+            "The accepted artifact is this content-hashed training report, its raw retained outputs, "
+            "raw independent-judge receipts, and explicit limitations.",
+            "",
+        ]
+    )
+
+
+def input_record(path: Path) -> dict[str, Any]:
+    return {
+        "path": str(path.relative_to(REPO_ROOT)),
+        "sha256": sha256_file(path),
+        "bytes": path.stat().st_size,
+    }
+
+
+def artifact_record(path: Path, run_dir: Path) -> dict[str, Any]:
+    return {
+        "path": str(path.relative_to(run_dir)),
+        "sha256": sha256_file(path),
+        "bytes": path.stat().st_size,
+    }
+
+
+def build_manifest(run_id: str, run_dir: Path, summary: dict[str, Any]) -> dict[str, Any]:
+    inputs = [
+        input_record(REPORT_PATH),
+        input_record(REPO_ROOT / "pyproject.toml"),
+        input_record(REPO_ROOT / "uv.lock"),
+        input_record(HERE / "run_training_report_audit.py"),
+        input_record(HERE / "validate_evidence.py"),
+    ]
+    artifact_paths = [
+        run_dir / "retained_outputs.json",
+        run_dir / "reproduction_contract.json",
+        run_dir / "judge_receipts.json",
+        run_dir / "summary.json",
+        run_dir / "report.md",
+    ]
+    return {
+        "schema_version": "exp8-3-manifest-v1",
+        "experiment": "8-3",
+        "run_id": run_id,
+        "created_at": utc_now(),
+        "status": summary["status"],
+        "run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
+        "inputs": inputs,
+        "artifacts": [artifact_record(path, run_dir) for path in artifact_paths],
+        "acceptance": summary["acceptance"],
+        "checkpoint_policy": "not distributed; not an acceptance artifact",
+    }
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--run-id", default=DEFAULT_RUN_ID)
+    parser.add_argument("--endpoint", default=os.getenv("ARK_BASE_URL", DEFAULT_ENDPOINT))
+    parser.add_argument("--model", default=os.getenv("ARK_MODEL", DEFAULT_MODEL))
+    parser.add_argument("--api-key-env", default="ARK_API_KEY")
+    parser.add_argument("--timeout", type=float, default=180.0)
+    parser.add_argument("--concurrency", type=int, default=8)
+    parser.add_argument(
+        "--refresh-manifest",
+        action="store_true",
+        help="Rehash an existing run after source-only corrections; makes no provider call.",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    if not re.fullmatch(r"[A-Za-z0-9._-]+", args.run_id):
+        raise SystemExit("run ID may contain only letters, digits, dot, underscore, and hyphen")
+    run_dir = RUNS_DIR / args.run_id
+    if args.refresh_manifest:
+        if not run_dir.is_dir():
+            raise SystemExit(f"cannot refresh missing run: {run_dir}")
+        stored_retained = json.loads(
+            (run_dir / "retained_outputs.json").read_text(encoding="utf-8")
+        )
+        current_retained = parse_retained_outputs()
+        # Documentation around the raw transcripts may change, but a refresh
+        # must never silently replace the generations that were judged.
+        if stored_retained.get("cells") != current_retained.get("cells"):
+            raise SystemExit(
+                "refusing manifest refresh because retained historical outputs changed"
+            )
+        stored_retained["source_report_sha256"] = current_retained[
+            "source_report_sha256"
+        ]
+        write_json(run_dir / "retained_outputs.json", stored_retained)
+        summary = json.loads((run_dir / "summary.json").read_text(encoding="utf-8"))
+        write_json(run_dir / "manifest.json", build_manifest(args.run_id, run_dir, summary))
+        latest = {
+            "schema_version": "exp8-3-latest-v1",
+            "experiment": "8-3",
+            "run_id": args.run_id,
+            "status": summary["status"],
+            "run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
+            "manifest_sha256": sha256_file(run_dir / "manifest.json"),
+        }
+        write_json(LATEST_PATH, latest)
+        print(json.dumps(latest, indent=2, sort_keys=True))
+        return 0
+
+    if run_dir.exists():
+        raise SystemExit(f"refusing to overwrite existing run: {run_dir}")
+    api_key = os.getenv(args.api_key_env)
+    if not api_key:
+        raise SystemExit(f"missing required credential environment variable: {args.api_key_env}")
+
+    retained = parse_retained_outputs()
+    comparisons = selected_comparisons(retained)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
+        futures = {
+            comparison["case_id"]: pool.submit(
+                call_judge,
+                comparison,
+                endpoint=args.endpoint,
+                model=args.model,
+                api_key=api_key,
+                timeout=args.timeout,
+            )
+            for comparison in comparisons
+        }
+        receipts = [futures[case_id].result() for case_id in sorted(futures)]
+
+    contract = reproduction_contract()
+    summary = summarize(retained, receipts, contract)
+    run_dir.mkdir(parents=True)
+    write_json(run_dir / "retained_outputs.json", retained)
+    write_json(
+        run_dir / "judge_receipts.json",
+        {
+            "schema_version": "exp8-3-judge-receipts-v1",
+            "experiment": "8-3",
+            "credential_headers_retained": False,
+            "calls": receipts,
+        },
+    )
+    write_json(run_dir / "reproduction_contract.json", contract)
+    write_json(run_dir / "summary.json", summary)
+    (run_dir / "report.md").write_text(render_report(summary), encoding="utf-8")
+    write_json(run_dir / "manifest.json", build_manifest(args.run_id, run_dir, summary))
+    latest = {
+        "schema_version": "exp8-3-latest-v1",
+        "experiment": "8-3",
+        "run_id": args.run_id,
+        "status": summary["status"],
+        "run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
+        "manifest_sha256": sha256_file(run_dir / "manifest.json"),
+    }
+    write_json(LATEST_PATH, latest)
+    print(json.dumps(latest, indent=2, sort_keys=True))
+    return 0 if summary["status"] == "passed" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())