liqiang/ai-agent-book
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
Details
dependency resolution / resolve (3.11) (push) Canceled after 0s
Details
dependency resolution / resolve (3.13) (push) Canceled after 0s
Details
deploy-pages / build (push) Canceled after 0s
Details
deploy-pages / deploy (push) Canceled after 0s
Details
i18n consistency check / check (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
Details
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
Details
web-search-agent tests / test (push) Canceled after 0s
Details
web-search-agent tests / agentbook (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
liqiang
2026-08-20 13:12:50 +00:00
commit b119135836
10275 changed files with 3284984 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chapter8/MiniMind-pretrain/README.md
+2063
View File
File diff suppressed because it is too large Load Diff
chapter8/MiniMind-pretrain/validation/latest.json
+8
View File
@@ -0,0 +1,8 @@
{
"experiment": "8-3",
"manifest_sha256": "2b99fb95727eba2d71cd1bf7e8a25585856efe0e61a7f02d3ff1bf07751fb9a3",
"run_dir": "validation/runs/exp8-3-training-report-20260731-v1",
"run_id": "exp8-3-training-report-20260731-v1",
"schema_version": "exp8-3-latest-v1",
"status": "passed"
}
chapter8/MiniMind-pretrain/validation/latest_vlm.json
+8
View File
@@ -0,0 +1,8 @@
{
"experiment": "8-4",
"manifest_sha256": "f984d549d8172528516d91800871df3dcd6b67f61729314b1371aa8022fabbf0",
"run_dir": "validation/runs/exp8-4-training-report-20260731-v1",
"run_id": "exp8-4-training-report-20260731-v1",
"schema_version": "exp8-4-latest-v1",
"status": "passed"
}
chapter8/MiniMind-pretrain/validation/run_training_report_audit.py
+760
View File
@@ -0,0 +1,760 @@
#!/usr/bin/env python3
"""Build checkpoint-free retained training evidence for Experiment 8-3.
The book already contains the author's historical six-cell evaluation report:
original versus QK-Norm + Muon at pretrain, SFT, and DPO. This program does
not pretend to rerun the GPU training job. It extracts every saved generation,
submits a preregistered stage-balanced subset to an arm-blind external judge,
and binds the raw report, judge receipts, immutable future-reproduction source
and dataset revisions, environment lock, findings, and limitations into a
content-hashed evidence package. Checkpoints are intentionally not published
and are not an acceptance artifact for book training experiments.
"""
from __future__ import annotations
import argparse
import concurrent.futures
import hashlib
import json
import os
import random
import re
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
HERE = Path(__file__).resolve().parent
EXPERIMENT_DIR = HERE.parent
REPO_ROOT = EXPERIMENT_DIR.parents[1]
REPORT_PATH = EXPERIMENT_DIR / "README.md"
RUNS_DIR = HERE / "runs"
LATEST_PATH = HERE / "latest.json"
DEFAULT_RUN_ID = "exp8-3-training-report-20260731-v1"
DEFAULT_ENDPOINT = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
DEFAULT_MODEL = "doubao-seed-1-6-250615"
BLIND_SEED = 730731
SOURCE_REVISION = "8bdc5d97d5845a8c1ac2ed56a5b8b4c0d0fb0795"
DATASET_REVISION = "84983ed4dec7836d240577760c1d6be5d4cabcf9"
SOURCE_FILES = {
"model/model_minimind.py": "2d33988711c704be6a22c4c61489b23106a2340a7cb8b97ebe3e40f30819cbb0",
"model/tokenizer.json": "e489029175fb3f94b8211a120a72a2ee41a664db65b828d077c7bde989c845a9",
"model/tokenizer_config.json": "190cc4738bac3b6f6b563376019c581b320fdb0260a03b9d5ab806296c8c6bb8",
"dataset/lm_dataset.py": "213726b1781289548784220b2b2db48fe97d84f3b67ac0cd70186cbbcb7b5d2c",
"trainer/muon.py": "00c2c6a225edeb55433df0724c3c74f6ff98ac4b2cc73c4aafcff686824f6267",
"trainer/train_pretrain.py": "ddd122645a9f1043bc8dac69a81ac51d2df95df8745d25faed7963d38fedc328",
"trainer/train_pretrain_muon.py": "fc83d07754ec3a8c156b6b8bfc0fd4326edecb72efabc5e08ae4ff5e3a7029bc",
"trainer/train_full_sft.py": "a57422f1df80bf2867f31f3b4a646a92ac7f66729e98a3b32cd1ec4d6780cb8b",
"trainer/train_full_sft_muon.py": "acd0b7db5b1d8b25d3c3103f92d68a7d381f9322a1b33bbec34be3d005930bad",
"trainer/train_dpo.py": "97f2c31cc8bc21a777e2efcb5e2fa35a49e4e9e3698db120148f8a0b2f678449",
"eval_model.py": "43930a4b55048a4a3ffa17eb78ae67d59582d639aa9365f0bbf41ba149128af8",
"requirements.txt": "23f4cea09281765eec7cf03e28231425638e8e42580f418781fed166b75af968",
}
DATASET_FILES = {
"pretrain_hq.jsonl": {
"lfs_sha256": "9801b0d2210c61c2e4bc130f6dc4b3c870698a88d04af8f103c23dd5f0ce2440",
"bytes": 1_669_750_047,
},
"sft_512.jsonl": {
"lfs_sha256": "053b7d09574e48a86232e929211434ff9e5016c6ed13312e63687dd52edcbebf",
"bytes": 7_531_517_862,
},
"dpo.jsonl": {
"lfs_sha256": "ee934a8a455ccc99d1334d63e1254dd1d64f497fd067cfcbb71e3043f5b46768",
"bytes": 53_653_322,
},
}
ARMS = ("original", "qk_norm_muon")
STAGES = ("pretrain", "sft", "dpo")
EXPECTED_COUNTS = {
("original", "pretrain"): 7,
("original", "sft"): 8,
("original", "dpo"): 9,
("qk_norm_muon", "pretrain"): 7,
("qk_norm_muon", "sft"): 9,
("qk_norm_muon", "dpo"): 9,
}
SELECTED_CASES = (
{"case_id": 1, "stage": "pretrain", "keyword": "highest mountain", "task": "Continue the prompt by identifying the highest mountain in the world accurately."},
{"case_id": 2, "stage": "pretrain", "keyword": "carbon dioxide", "task": "Continue the prompt with an accurate statement about carbon dioxide in air."},
{"case_id": 3, "stage": "sft", "keyword": "speed of light", "task": "Explain the physical concept of the speed of light in detail."},
{"case_id": 4, "stage": "sft", "keyword": "how to understand chatgpt", "task": "Explain what ChatGPT is and how it works."},
{"case_id": 5, "stage": "sft", "keyword": "history of the united states", "task": "Introduce the history of the United States."},
{"case_id": 6, "stage": "dpo", "keyword": "speed of light", "task": "Explain the physical concept of the speed of light in detail."},
{"case_id": 7, "stage": "dpo", "keyword": "how to understand chatgpt", "task": "Explain what ChatGPT is and how it works."},
{"case_id": 8, "stage": "dpo", "keyword": "history of the united states", "task": "Introduce the history of the United States."},
)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def canonical_json_bytes(value: Any) -> bytes:
return (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode()
def write_json(path: Path, value: Any) -> None:
path.write_bytes(canonical_json_bytes(value))
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def _terminal_transcript(section: str) -> tuple[str, str]:
"""Return the terminal header and prompt/output transcript from one cell.
The historical Markdown has one stray closing fence immediately before the
improved-SFT Lu Xun answer. Treat fences as presentation markup rather
than semantic delimiters so that the retained answer is not silently lost.
"""
prompt_position = section.find("👶:")
if prompt_position < 0:
raise ValueError("model section has no user prompt marker")
header = section[:prompt_position].replace("```", "").strip()
transcript = section[prompt_position:]
analysis = re.search(r"(?m)^\*\*[^\n]*Analysis[^\n]*\*\*:?\s*$", transcript)
if analysis:
transcript = transcript[: analysis.start()]
transcript = re.sub(r"(?m)^```\s*$", "", transcript).strip()
return header, transcript
def _parse_pairs(transcript: str) -> list[dict[str, str]]:
pattern = re.compile(
r"^👶:\s*(.*?)\n🤖️:\s*(.*?)(?=\n(?:\s*\n)*👶:|\Z)",
flags=re.DOTALL | re.MULTILINE,
)
pairs = []
for match in pattern.finditer(transcript):
prompt = match.group(1).strip()
output = match.group(2).strip()
if not prompt or not output:
raise ValueError("empty prompt or output in historical transcript")
pairs.append({"prompt": prompt, "output": output})
return pairs
def parse_retained_outputs(report_path: Path = REPORT_PATH) -> dict[str, Any]:
"""Extract all six historical LLM evaluation cells from the bilingual report."""
text = report_path.read_text(encoding="utf-8")
start = text.index("## Language Model Training Results Analysis")
end = text.index("# Analysis of Vision-Language Model Training Results", start)
llm_text = text[start:end]
arm_markers = {
"original": "## Without Muon Optimizer (Original Architecture)",
"qk_norm_muon": "## With Muon Optimizer and QK Norm (Improved Architecture)",
}
cells: list[dict[str, Any]] = []
for arm_index, arm in enumerate(ARMS):
arm_start = llm_text.index(arm_markers[arm])
arm_end = (
llm_text.index(arm_markers[ARMS[arm_index + 1]], arm_start)
if arm_index + 1 < len(ARMS)
else len(llm_text)
)
arm_text = llm_text[arm_start:arm_end]
for stage_index, stage in enumerate(STAGES):
heading = {"pretrain": "### Pretrain Model", "sft": "### SFT Model", "dpo": "### DPO Model"}[stage]
cell_start = arm_text.index(heading)
next_positions = [
arm_text.find(next_heading, cell_start + len(heading))
for next_heading in ("### Pretrain Model", "### SFT Model", "### DPO Model")
]
next_positions = [position for position in next_positions if position >= 0]
cell_end = min(next_positions) if next_positions else len(arm_text)
header, transcript = _terminal_transcript(arm_text[cell_start:cell_end])
pairs = _parse_pairs(transcript)
expected = EXPECTED_COUNTS[(arm, stage)]
if len(pairs) != expected:
raise ValueError(f"{arm}/{stage}: expected {expected} pairs, found {len(pairs)}")
cells.append(
{
"arm": arm,
"stage": stage,
"terminal_header": header,
"pair_count": len(pairs),
"pairs": pairs,
}
)
return {
"schema_version": "exp8-3-retained-outputs-v1",
"experiment": "8-3",
"source_report": str(report_path.relative_to(REPO_ROOT)),
"source_report_sha256": sha256_file(report_path),
"arms": list(ARMS),
"stages": list(STAGES),
"cell_count": len(cells),
"output_count": sum(cell["pair_count"] for cell in cells),
"cells": cells,
}
def _find_pair(retained: dict[str, Any], arm: str, stage: str, keyword: str) -> dict[str, str]:
cell = next(cell for cell in retained["cells"] if cell["arm"] == arm and cell["stage"] == stage)
matches = [pair for pair in cell["pairs"] if keyword in pair["prompt"].lower()]
if len(matches) != 1:
raise ValueError(f"{arm}/{stage}/{keyword}: expected one prompt, found {len(matches)}")
return matches[0]
def selected_comparisons(retained: dict[str, Any]) -> list[dict[str, Any]]:
comparisons = []
for case in SELECTED_CASES:
rows = {
arm: _find_pair(retained, arm, case["stage"], case["keyword"])
for arm in ARMS
}
comparisons.append({**case, "arms": rows})
return comparisons
def blind_mapping(case_id: int) -> dict[str, str]:
arms = list(ARMS)
random.Random(BLIND_SEED + case_id).shuffle(arms)
return dict(zip(("A", "B"), arms, strict=True))
def judge_payload(comparison: dict[str, Any], mapping: dict[str, str], model: str) -> dict[str, Any]:
candidates = {
label: {
"historical_prompt": comparison["arms"][arm]["prompt"],
"historical_output": comparison["arms"][arm]["output"],
}
for label, arm in mapping.items()
}
required = {
"case_id": comparison["case_id"],
"candidates": {
label: {
"language_fluency": "number 0-5",
"instruction_following": "number 0-5",
"factuality": "number 0-5",
"factual_errors": ["specific material errors; empty only if none"],
"rationale": "brief evidence-based explanation",
}
for label in ("A", "B")
},
"winner": "A, B, or tie",
}
return {
"model": model,
"temperature": 0,
"response_format": {"type": "json_object"},
"messages": [
{
"role": "system",
"content": (
"You are an independent evaluator of two anonymous small-language-model outputs. "
"Do not infer model identity, architecture, optimizer, or training stage. Score only "
"the supplied text. Penalize hallucinations, unsafe medical specificity, repetition, "
"and non-answers. Return one JSON object with exactly the requested fields."
),
},
{
"role": "user",
"content": json.dumps(
{
"case_id": comparison["case_id"],
"task": comparison["task"],
"rubric": {
"language_fluency": "0 unreadable; 3 understandable with defects; 5 coherent and natural",
"instruction_following": "0 non-answer; 3 partial; 5 directly and fully answers",
"factuality": "0 dominated by falsehoods; 3 mixed/minor errors; 5 no material error",
},
"candidates": candidates,
"required_json_shape": required,
},
ensure_ascii=False,
sort_keys=True,
),
},
],
}
def extract_json_object(content: str) -> dict[str, Any]:
stripped = content.strip()
if stripped.startswith("```"):
stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
stripped = re.sub(r"\s*```$", "", stripped)
parsed = json.loads(stripped)
if not isinstance(parsed, dict):
raise ValueError("judge content must decode to an object")
return parsed
def validate_judgment(judgment: dict[str, Any], case_id: int) -> None:
if str(judgment.get("case_id")) != str(case_id):
raise ValueError(
f"judge returned the wrong case_id: expected {case_id}, got {judgment.get('case_id')!r}"
)
candidates = judgment.get("candidates")
if not isinstance(candidates, dict) or set(candidates) != {"A", "B"}:
raise ValueError("judge must score A and B exactly")
for label in ("A", "B"):
row = candidates[label]
if not isinstance(row, dict):
raise ValueError(f"candidate {label} score must be an object")
for metric in ("language_fluency", "instruction_following", "factuality"):
score = row.get(metric)
if not isinstance(score, (int, float)) or isinstance(score, bool) or not 0 <= score <= 5:
raise ValueError(f"candidate {label} has invalid {metric}: {score!r}")
if not isinstance(row.get("factual_errors"), list):
raise ValueError(f"candidate {label} factual_errors must be a list")
if not isinstance(row.get("rationale"), str) or not row["rationale"].strip():
raise ValueError(f"candidate {label} rationale is missing")
if judgment.get("winner") not in {"A", "B", "tie"}:
raise ValueError("judge winner must be A, B, or tie")
def call_judge(
comparison: dict[str, Any], *, endpoint: str, model: str, api_key: str, timeout: float
) -> dict[str, Any]:
mapping = blind_mapping(comparison["case_id"])
payload = judge_payload(comparison, mapping, model)
request = urllib.request.Request(
endpoint,
data=json.dumps(payload, ensure_ascii=False).encode(),
method="POST",
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
)
started = time.perf_counter()
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
raw_body = response.read()
http_status = response.status
except urllib.error.HTTPError as exc:
body = exc.read().decode(errors="replace")
raise RuntimeError(f"judge HTTP {exc.code}: {body[:500]}") from exc
latency_ms = round((time.perf_counter() - started) * 1000, 3)
raw_response = json.loads(raw_body)
content = raw_response["choices"][0]["message"]["content"]
judgment = extract_json_object(content)
validate_judgment(judgment, comparison["case_id"])
response_id = raw_response.get("id")
usage = raw_response.get("usage")
if not isinstance(response_id, str) or not response_id:
raise ValueError("judge response has no response ID")
if not isinstance(usage, dict) or not isinstance(usage.get("total_tokens"), int):
raise ValueError("judge response has no complete usage object")
return {
"case_id": comparison["case_id"],
"stage": comparison["stage"],
"keyword": comparison["keyword"],
"task": comparison["task"],
"provider": "ark",
"endpoint": endpoint,
"credential_env": "ARK_API_KEY",
"credential_headers_retained": False,
"blind_seed": BLIND_SEED,
"blind_map": mapping,
"request": payload,
"http_status": http_status,
"response": raw_response,
"response_id": response_id,
"usage": usage,
"latency_ms": latency_ms,
"judgment": judgment,
}
def reproduction_contract() -> dict[str, Any]:
return {
"schema_version": "exp8-3-reproduction-contract-v1",
"experiment": "8-3",
"historical_evidence_boundary": {
"historical_training_executed": True,
"six_historical_evaluation_transcripts_retained": True,
"historical_source_revision_retained": False,
"historical_dataset_hashes_retained": False,
"historical_checkpoint_hashes_retained": False,
"historical_stepwise_training_logs_retained": False,
"claim": (
"The author's retained report supports that original and QK-Norm+Muon 104.03M models "
"were evaluated after pretrain, SFT, and DPO. It does not establish byte identity of the "
"historical checkpoints, datasets, source checkout, or every loss point."
),
},
"future_reproduction": {
"source": {
"repository": "bojieli/minimind",
"revision": SOURCE_REVISION,
"selected_at": "2026-07-31",
"not_claimed_as_historical_revision": True,
"file_sha256": SOURCE_FILES,
},
"dataset": {
"repository": "jingyaogong/minimind_dataset",
"revision": DATASET_REVISION,
"selected_at": "2026-07-31",
"not_claimed_as_historical_revision": True,
"files": DATASET_FILES,
},
"commands": {
"original_pretrain": "torchrun --nproc_per_node=8 trainer/train_pretrain.py --epochs 10 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/pretrain_hq.jsonl --use_wandb",
"improved_pretrain": "torchrun --nproc_per_node=8 trainer/train_pretrain_muon.py --epochs 10 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/pretrain_hq.jsonl --use_wandb",
"original_sft": "torchrun --nproc_per_node=8 trainer/train_full_sft.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/sft_512.jsonl --use_wandb",
"improved_sft": "torchrun --nproc_per_node=8 trainer/train_full_sft_muon.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/sft_512.jsonl --use_wandb",
"original_dpo": "torchrun --nproc_per_node=8 trainer/train_dpo.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --init_from out/full_sft_768.pth --data_path dataset/dpo.jsonl --use_wandb",
"improved_dpo": "torchrun --nproc_per_node=8 trainer/train_dpo.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --init_from out/full_sft_muon_768.pth --data_path dataset/dpo.jsonl --use_wandb",
},
"environment": {
"book_pyproject": "pyproject.toml",
"book_lock": "uv.lock",
"install": "uv sync --locked --python 3.12 --extra ch7 --extra dev",
"boundary": (
"The book lock freezes a future software environment. The pinned upstream requirements "
"file itself is unversioned, and the future GPU/CUDA stack has not been exercised here."
),
},
},
"model_and_training": {
"reported_parameter_count_millions": 104.03,
"architecture": {"hidden_size": 768, "layers": 16, "sequence_length": 512},
"stages": list(STAGES),
"arms": list(ARMS),
"source_verified_mechanisms": {
"qk_norm_before_rope": True,
"muon_for_two_dimensional_non_embedding_weights": True,
"adamw_for_embeddings_norms_and_lm_head": True,
"dpo_uses_adamw_from_arm_specific_sft_checkpoint": True,
},
"reported_scalars_without_stepwise_logs": {
"steps_to_loss_3_original": 36,
"steps_to_loss_3_qk_norm_muon": 12,
"final_loss_original": 2.0,
"final_loss_qk_norm_muon": 1.7,
"eight_rtx_4090_pretrain_hours": 6,
"eight_rtx_4090_sft_hours": 8,
},
},
"checkpoint_policy": {
"distributed_with_book": False,
"acceptance_artifact": False,
"required_artifact": "reproducible evidence-backed training report",
"reason": "Training checkpoints are intentionally not distributed to readers.",
},
}
def mean(values: list[float]) -> float:
return round(sum(values) / len(values), 4)
def summarize(retained: dict[str, Any], receipts: list[dict[str, Any]], contract: dict[str, Any]) -> dict[str, Any]:
metrics = ("language_fluency", "instruction_following", "factuality")
rows: dict[int, dict[str, dict[str, Any]]] = {}
wins = {"original": 0, "qk_norm_muon": 0, "tie": 0}
for receipt in receipts:
reverse = receipt["blind_map"]
rows[receipt["case_id"]] = {
reverse[label]: score for label, score in receipt["judgment"]["candidates"].items()
}
winner = receipt["judgment"]["winner"]
wins["tie" if winner == "tie" else reverse[winner]] += 1
arm_averages = {}
stage_averages = {}
for arm in ARMS:
arm_scores = [rows[case["case_id"]][arm] for case in SELECTED_CASES]
arm_averages[arm] = {
metric: mean([float(row[metric]) for row in arm_scores]) for metric in metrics
}
arm_averages[arm]["overall"] = mean(
[float(row[metric]) for row in arm_scores for metric in metrics]
)
stage_averages[arm] = {}
for stage in STAGES:
stage_scores = [
rows[case["case_id"]][arm] for case in SELECTED_CASES if case["stage"] == stage
]
stage_averages[arm][stage] = {
metric: mean([float(row[metric]) for row in stage_scores]) for metric in metrics
}
stage_averages[arm][stage]["overall"] = mean(
[float(row[metric]) for row in stage_scores for metric in metrics]
)
findings = {
"blind_judge_overall_delta_qk_norm_muon_minus_original": round(
arm_averages["qk_norm_muon"]["overall"] - arm_averages["original"]["overall"], 4
),
"blind_judge_prefers_qk_norm_muon_overall": (
arm_averages["qk_norm_muon"]["overall"] > arm_averages["original"]["overall"]
),
"wins": wins,
"reported_loss_comparison_retained_but_not_independently_recomputed": True,
}
acceptance = {
"raw_historical_report_hashed": bool(retained["source_report_sha256"]),
"all_six_arm_stage_cells_retained": retained["cell_count"] == 6,
"all_expected_outputs_retained": retained["output_count"] == sum(EXPECTED_COUNTS.values()),
"pretrain_sft_and_dpo_compared": set(retained["stages"]) == set(STAGES),
"original_and_qk_norm_muon_compared": set(retained["arms"]) == set(ARMS),
"eight_stage_balanced_blind_judgments": len(receipts) == len(SELECTED_CASES),
"raw_judge_requests_responses_ids_usage_latency_retained": all(
receipt["response_id"]
and receipt["usage"].get("total_tokens", 0) > 0
and receipt["latency_ms"] > 0
for receipt in receipts
),
"immutable_source_revision_and_file_hashes_frozen": bool(SOURCE_REVISION and SOURCE_FILES),
"immutable_dataset_revision_lfs_hashes_and_sizes_frozen": bool(DATASET_REVISION and DATASET_FILES),
"future_reproduction_commands_declared": len(contract["future_reproduction"]["commands"]) == 6,
"historical_provenance_limitations_explicit": (
contract["historical_evidence_boundary"]["historical_checkpoint_hashes_retained"] is False
and contract["historical_evidence_boundary"]["historical_stepwise_training_logs_retained"] is False
),
"reported_loss_claims_qualified": findings[
"reported_loss_comparison_retained_but_not_independently_recomputed"
],
"checkpoints_not_an_acceptance_artifact": (
contract["checkpoint_policy"]["acceptance_artifact"] is False
),
}
passed = all(acceptance.values())
return {
"schema_version": "exp8-3-summary-v1",
"experiment": "8-3",
"status": "passed" if passed else "failed",
"judge": {
"provider": "ark",
"model": receipts[0]["request"]["model"],
"calls": len(receipts),
"response_ids": [receipt["response_id"] for receipt in receipts],
"total_tokens": sum(receipt["usage"]["total_tokens"] for receipt in receipts),
"total_latency_ms": round(sum(receipt["latency_ms"] for receipt in receipts), 3),
"blind_seed": BLIND_SEED,
},
"retained": {
"cells": retained["cell_count"],
"outputs": retained["output_count"],
"selected_comparisons": len(receipts),
},
"arm_averages": arm_averages,
"stage_averages": stage_averages,
"per_case_arm_scores": rows,
"scientific_findings": findings,
"acceptance": {**acceptance, "passed": passed},
"limitations": [
"Historical checkpoints are intentionally not distributed and were not recreated in this audit.",
"The historical source revision, dataset byte identities, RNG state, and stepwise loss logs were not retained.",
"Frozen source/data revisions and the book lock define a future reproduction contract, not historical provenance.",
"The independent judge covers eight preregistered comparisons; all other retained outputs remain available for inspection.",
"The historical outputs are English translations in a bilingual report, so translation may affect the judge scores.",
],
}
def render_report(summary: dict[str, Any]) -> str:
averages = summary["arm_averages"]
findings = summary["scientific_findings"]
return "\n".join(
[
"# Experiment 8-3 retained-training-report audit",
"",
"## Result",
"",
f"Status: **{summary['status']}**. The historical report retains "
f"{summary['retained']['outputs']} outputs across the original and QK-Norm + Muon arms "
"after pretrain, SFT, and DPO. Eight preregistered arm-blind comparisons were judged "
"from raw retained text by an independent ARK model.",
"",
"| Arm | Fluency | Instruction | Factuality | Overall |",
"| --- | ---: | ---: | ---: | ---: |",
f"| Original | {averages['original']['language_fluency']:.4f} | "
f"{averages['original']['instruction_following']:.4f} | "
f"{averages['original']['factuality']:.4f} | {averages['original']['overall']:.4f} |",
f"| QK-Norm + Muon | {averages['qk_norm_muon']['language_fluency']:.4f} | "
f"{averages['qk_norm_muon']['instruction_following']:.4f} | "
f"{averages['qk_norm_muon']['factuality']:.4f} | {averages['qk_norm_muon']['overall']:.4f} |",
"",
f"Observed blind-judge overall delta: **{findings['blind_judge_overall_delta_qk_norm_muon_minus_original']:+.4f}**. "
f"Pairwise decisions: {findings['wins']}.",
"",
"The report's loss claims (3.0 reached at 36 versus 12 reported steps; final loss 2.0 "
"versus 1.7) are retained as author-reported observations, not independently recomputed "
"measurements, because the historical stepwise logs were not preserved.",
"",
"## Provenance and reproduction boundary",
"",
"`reproduction_contract.json` freezes the MiniMind source revision, hashes the relevant "
"source files, freezes a dataset revision with the three Git-LFS object hashes and sizes, "
"and records all six future reproduction commands. These pins were selected for future "
"reproduction and are not represented as the exact historical checkout.",
"",
"Training checkpoints remain local by book policy and are not an acceptance artifact. "
"The accepted artifact is this content-hashed training report, its raw retained outputs, "
"raw independent-judge receipts, and explicit limitations.",
"",
]
)
def input_record(path: Path) -> dict[str, Any]:
return {
"path": str(path.relative_to(REPO_ROOT)),
"sha256": sha256_file(path),
"bytes": path.stat().st_size,
}
def artifact_record(path: Path, run_dir: Path) -> dict[str, Any]:
return {
"path": str(path.relative_to(run_dir)),
"sha256": sha256_file(path),
"bytes": path.stat().st_size,
}
def build_manifest(run_id: str, run_dir: Path, summary: dict[str, Any]) -> dict[str, Any]:
inputs = [
input_record(REPORT_PATH),
input_record(REPO_ROOT / "pyproject.toml"),
input_record(REPO_ROOT / "uv.lock"),
input_record(HERE / "run_training_report_audit.py"),
input_record(HERE / "validate_evidence.py"),
]
artifact_paths = [
run_dir / "retained_outputs.json",
run_dir / "reproduction_contract.json",
run_dir / "judge_receipts.json",
run_dir / "summary.json",
run_dir / "report.md",
]
return {
"schema_version": "exp8-3-manifest-v1",
"experiment": "8-3",
"run_id": run_id,
"created_at": utc_now(),
"status": summary["status"],
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
"inputs": inputs,
"artifacts": [artifact_record(path, run_dir) for path in artifact_paths],
"acceptance": summary["acceptance"],
"checkpoint_policy": "not distributed; not an acceptance artifact",
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--run-id", default=DEFAULT_RUN_ID)
parser.add_argument("--endpoint", default=os.getenv("ARK_BASE_URL", DEFAULT_ENDPOINT))
parser.add_argument("--model", default=os.getenv("ARK_MODEL", DEFAULT_MODEL))
parser.add_argument("--api-key-env", default="ARK_API_KEY")
parser.add_argument("--timeout", type=float, default=180.0)
parser.add_argument("--concurrency", type=int, default=8)
parser.add_argument(
"--refresh-manifest",
action="store_true",
help="Rehash an existing run after source-only corrections; makes no provider call.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
if not re.fullmatch(r"[A-Za-z0-9._-]+", args.run_id):
raise SystemExit("run ID may contain only letters, digits, dot, underscore, and hyphen")
run_dir = RUNS_DIR / args.run_id
if args.refresh_manifest:
if not run_dir.is_dir():
raise SystemExit(f"cannot refresh missing run: {run_dir}")
stored_retained = json.loads(
(run_dir / "retained_outputs.json").read_text(encoding="utf-8")
)
current_retained = parse_retained_outputs()
# Documentation around the raw transcripts may change, but a refresh
# must never silently replace the generations that were judged.
if stored_retained.get("cells") != current_retained.get("cells"):
raise SystemExit(
"refusing manifest refresh because retained historical outputs changed"
)
stored_retained["source_report_sha256"] = current_retained[
"source_report_sha256"
]
write_json(run_dir / "retained_outputs.json", stored_retained)
summary = json.loads((run_dir / "summary.json").read_text(encoding="utf-8"))
write_json(run_dir / "manifest.json", build_manifest(args.run_id, run_dir, summary))
latest = {
"schema_version": "exp8-3-latest-v1",
"experiment": "8-3",
"run_id": args.run_id,
"status": summary["status"],
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
"manifest_sha256": sha256_file(run_dir / "manifest.json"),
}
write_json(LATEST_PATH, latest)
print(json.dumps(latest, indent=2, sort_keys=True))
return 0
if run_dir.exists():
raise SystemExit(f"refusing to overwrite existing run: {run_dir}")
api_key = os.getenv(args.api_key_env)
if not api_key:
raise SystemExit(f"missing required credential environment variable: {args.api_key_env}")
retained = parse_retained_outputs()
comparisons = selected_comparisons(retained)
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futures = {
comparison["case_id"]: pool.submit(
call_judge,
comparison,
endpoint=args.endpoint,
model=args.model,
api_key=api_key,
timeout=args.timeout,
)
for comparison in comparisons
}
receipts = [futures[case_id].result() for case_id in sorted(futures)]
contract = reproduction_contract()
summary = summarize(retained, receipts, contract)
run_dir.mkdir(parents=True)
write_json(run_dir / "retained_outputs.json", retained)
write_json(
run_dir / "judge_receipts.json",
{
"schema_version": "exp8-3-judge-receipts-v1",
"experiment": "8-3",
"credential_headers_retained": False,
"calls": receipts,
},
)
write_json(run_dir / "reproduction_contract.json", contract)
write_json(run_dir / "summary.json", summary)
(run_dir / "report.md").write_text(render_report(summary), encoding="utf-8")
write_json(run_dir / "manifest.json", build_manifest(args.run_id, run_dir, summary))
latest = {
"schema_version": "exp8-3-latest-v1",
"experiment": "8-3",
"run_id": args.run_id,
"status": summary["status"],
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
"manifest_sha256": sha256_file(run_dir / "manifest.json"),
}
write_json(LATEST_PATH, latest)
print(json.dumps(latest, indent=2, sort_keys=True))
return 0 if summary["status"] == "passed" else 1
if __name__ == "__main__":
raise SystemExit(main())
chapter8/MiniMind-pretrain/validation/run_vlm_training_report_audit.py
Executable
+922
View File
@@ -0,0 +1,922 @@
#!/usr/bin/env python3
"""Build checkpoint-free retained training evidence for Experiment 8-4.
The book contains 64 historical MiniMind-V image descriptions: eight model
configurations evaluated on the same eight images. This program extracts all
of them, asks a real image-capable model to judge every anonymous candidate
against the corresponding source image, and writes raw credential-free
requests/responses plus a fail-closed, content-hashed reproduction package.
It deliberately does not claim to rerun the historical GPU jobs. Historical
checkpoints are intentionally not distributed; the accepted artifact is a
reproducible training report with explicit provenance limits.
"""
from __future__ import annotations
import argparse
import base64
import concurrent.futures
import hashlib
import json
import mimetypes
import os
import random
import re
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
HERE = Path(__file__).resolve().parent
EXPERIMENT_DIR = HERE.parent
REPO_ROOT = EXPERIMENT_DIR.parents[1]
REPORT_PATH = EXPERIMENT_DIR / "README.md"
RUNS_DIR = HERE / "runs"
LATEST_PATH = HERE / "latest_vlm.json"
DEFAULT_RUN_ID = "exp8-4-training-report-20260731-v1"
DEFAULT_ENDPOINT = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
DEFAULT_MODEL = "doubao-seed-1-6-250615"
BLIND_SEED = 740731
ORIGINAL_VLM_REVISION = "765908051d0837d60cecfb93f8390334e2e55f1e"
IMPROVED_VLM_REVISION = "ead791c530fa5f9a3549dbfe9e11ec732d18d2e5"
ORIGINAL_LLM_REVISION = "6d160ea20b98324632c4447ee63ec7cfa9becd20"
IMPROVED_LLM_REVISION = "8bdc5d97d5845a8c1ac2ed56a5b8b4c0d0fb0795"
DATASET_REVISION = "ac9d03a3fd26a2d8e74bda374d9a2ddba49e4c1b"
CLIP_REVISION = "57c216476eefef5ab752ec549e440a49ae4ae5f3"
ORIGINAL_VLM_FILES = {
"trainer/train_pretrain_vlm.py": "4d30d54a940ae2eced204971cc03aafb3eb41a5c84c9f033cfdf162e63924a4d",
"trainer/train_sft_vlm.py": "8e3b920a6a135eb126bbeea07e2db748729cdd86050925b282a80537bc324e5f",
"eval_vlm.py": "9d883e4adbab0a7b88fd0cb9034132559a365387ec273ac4811cdd5ad28d5cda",
"model/model_minimind.py": "105429e93dcbe87145264d72d46a6add7639666036e999628c76ae50582507dc",
"model/model_vlm.py": "4ee42b298db68f30fbfa06d7686aa375d41a697628c770d0a134bca40ca9ea80",
"dataset/lm_dataset.py": "df20d57460d2845841ddf2e0faced1af1f7ec169e7fda3cd50fe3b2854288a92",
"model/tokenizer.json": "d98595c6aef70d95f72748582fb9b4f53d76dd58c1ae1dd702ad7c84e1caf5e4",
"model/tokenizer_config.json": "dbbdb7eea33aba5c2608471494c93f650a2cf46fbe4a7489e531537ddadee746",
"requirements.txt": "a9bddf49d3ccbc9f8a2508ea039aebc0b996dccb0d3618d1b119af53a5d49869",
}
IMPROVED_VLM_FILES = {
"trainer/train_pretrain_vlm_muon.py": "f39af354c588747d9d5e522c9374a7f59a35d57aa649da74957da67d95d25bc6",
"trainer/train_sft_vlm_muon.py": "1fd56b3e8bed2714b4d10ceba5d57ada0b95d00dfbfb514481498fef0c0dd03d",
"trainer/muon.py": "00c2c6a225edeb55433df0724c3c74f6ff98ac4b2cc73c4aafcff686824f6267",
"eval_vlm.py": "9d883e4adbab0a7b88fd0cb9034132559a365387ec273ac4811cdd5ad28d5cda",
"model/model_minimind.py": "4771bc4b2ac367a6e6415c42c30bcdb54bec0397708f87de3c390042680b1e9e",
"model/model_vlm.py": "4ee42b298db68f30fbfa06d7686aa375d41a697628c770d0a134bca40ca9ea80",
"dataset/lm_dataset.py": "df20d57460d2845841ddf2e0faced1af1f7ec169e7fda3cd50fe3b2854288a92",
"model/tokenizer.json": "e489029175fb3f94b8211a120a72a2ee41a664db65b828d077c7bde989c845a9",
"model/tokenizer_config.json": "190cc4738bac3b6f6b563376019c581b320fdb0260a03b9d5ab806296c8c6bb8",
"requirements.txt": "a9bddf49d3ccbc9f8a2508ea039aebc0b996dccb0d3618d1b119af53a5d49869",
}
ORIGINAL_LLM_FILES = {
"model/model_minimind.py": "7cb069cb0cb0dfa123cf11ea394d0001270bc683c0a2dfe4120fc3b861ffc0a4",
"trainer/train_pretrain.py": "ddd122645a9f1043bc8dac69a81ac51d2df95df8745d25faed7963d38fedc328",
"trainer/train_full_sft.py": "a57422f1df80bf2867f31f3b4a646a92ac7f66729e98a3b32cd1ec4d6780cb8b",
"trainer/train_dpo.py": "5e556a3089e43681638cdbf5adafb9d085bb1de5e4ea8da3ee522dfae02e3599",
"eval_model.py": "b9f7ea9d7f517551362bbf2da8f1de006b8c734bcba774b2be752bc63cc4349d",
}
IMPROVED_LLM_FILES = {
"model/model_minimind.py": "2d33988711c704be6a22c4c61489b23106a2340a7cb8b97ebe3e40f30819cbb0",
"trainer/train_pretrain_muon.py": "fc83d07754ec3a8c156b6b8bfc0fd4326edecb72efabc5e08ae4ff5e3a7029bc",
"trainer/train_full_sft_muon.py": "acd0b7db5b1d8b25d3c3103f92d68a7d381f9322a1b33bbec34be3d005930bad",
"trainer/train_dpo.py": "97f2c31cc8bc21a777e2efcb5e2fa35a49e4e9e3698db120148f8a0b2f678449",
"eval_model.py": "43930a4b55048a4a3ffa17eb78ae67d59582d639aa9365f0bbf41ba149128af8",
}
DATASET_FILES = {
"pretrain_data.jsonl": {
"lfs_sha256": "abc9f2ba44190646692fbe7e2b49c366c5045490989fb32d2c5e960dd0ee10e4",
"bytes": 134315765,
},
"pretrain_images.zip": {
"lfs_sha256": "64d56cee145bed75bc7f94c9cbf58882c41c4a0fea993014e27de7490b49e8b7",
"bytes": 2614907051,
},
"sft_data.jsonl": {
"lfs_sha256": "c1993d38c3a22a8bdfee65affc82d6559e5bb62e785b0f21c9151c75116151fc",
"bytes": 173137988,
},
"sft_images.zip": {
"lfs_sha256": "89ee34facc6793c51613613e0b10cac078942282f5fdec48d85751c6224bc3c2",
"bytes": 1026332147,
},
}
CLIP_FILE = {
"path": "pytorch_model.bin",
"lfs_sha256": "ec89c7b09c749a60aae3c9cd910516f24b58214a7df060b48962d14c469cfbf0",
"bytes": 598641023,
}
IMAGE_FILES = {
"Rainbow-Falls.jpg": "彩虹瀑布-Rainbow-Falls.jpg",
"Dog-Woman-Sea.jpg": "小狗美女海边-Dog-Woman-Sea.jpg",
"dance.jpg": "舞蹈-dance.jpg",
"Astronaut-Space.jpg": "太空宇航员-Astronaut-Space.jpg",
"city-traffic.jpg": "城市车水马龙-city-traffic.jpg",
"Panda-Grassland.jpg": "熊猫草地-Panda-Grassland.jpg",
"Bicycle-Flowers.jpg": "自行车鲜花-Bicycle-Flowers.jpg",
"Chair-Elderly-Reading.jpg": "椅子老人看书-Chair-Elderly-Reading.jpg",
}
IMAGE_SHA256 = {
"Rainbow-Falls.jpg": "1c8b74debaceb2e0bb6171b182084afe49288a0cc8089eb91eac69d067c27b10",
"Dog-Woman-Sea.jpg": "ba90d8b8738a44eac70811be5c89f767492b167ad4f6f6c31aa4591837d7e3dc",
"dance.jpg": "939e3132c8d3aec81f66f8aa928b476aaa25e00d94f1097f4974e73c913d5d8c",
"Astronaut-Space.jpg": "f466cdafecbdb85d2bad586896db5db3313afe18f9b3505667756cd25b747747",
"city-traffic.jpg": "73e90d82fbc5b1cf43b40de782b443f93f43a34e66b8ddebf3146d5dc1f83e00",
"Panda-Grassland.jpg": "0b7610a881039f0effdbfa46e9bb189132443d3ce2956856e8adf66d1ca22f8c",
"Bicycle-Flowers.jpg": "44fae0fafcd52c20b9bcaded897facbff00f61019cdd0aea543addf8499ad899",
"Chair-Elderly-Reading.jpg": "8fe91a90e837c33230d21cfe7ba5020e71b3ae99ac4c3fbd6d32cb54f51def53",
}
CONFIGS = (
"without_muon_pretrained",
"without_muon_sft",
"muon_from_dpo_pretrained",
"muon_from_dpo_sft",
"muon_from_pretrain_pretrained",
"muon_from_pretrain_sft",
"muon_from_sft_pretrained",
"muon_from_sft_sft",
)
CONFIG_META = {
"without_muon_pretrained": {
"architecture": "original",
"base_llm_stage": "sft",
"vlm_stage": "pretrained",
},
"without_muon_sft": {"architecture": "original", "base_llm_stage": "sft", "vlm_stage": "sft"},
"muon_from_dpo_pretrained": {
"architecture": "qk_norm_muon",
"base_llm_stage": "dpo",
"vlm_stage": "pretrained",
},
"muon_from_dpo_sft": {
"architecture": "qk_norm_muon",
"base_llm_stage": "dpo",
"vlm_stage": "sft",
},
"muon_from_pretrain_pretrained": {
"architecture": "qk_norm_muon",
"base_llm_stage": "pretrain",
"vlm_stage": "pretrained",
},
"muon_from_pretrain_sft": {
"architecture": "qk_norm_muon",
"base_llm_stage": "pretrain",
"vlm_stage": "sft",
},
"muon_from_sft_pretrained": {
"architecture": "qk_norm_muon",
"base_llm_stage": "sft",
"vlm_stage": "pretrained",
},
"muon_from_sft_sft": {
"architecture": "qk_norm_muon",
"base_llm_stage": "sft",
"vlm_stage": "sft",
},
}
SECTION_SPECS = (
(
"## Without Muon Optimizer",
(
("without_muon_pretrained", "### Pretrained VLM"),
("without_muon_sft", "### VLM after SFT"),
),
),
(
"## VLM with Muon Optimizer (from DPO)",
(
("muon_from_dpo_pretrained", "### Pretrained VLM"),
("muon_from_dpo_sft", "### VLM with SFT"),
),
),
(
"## VLM with Muon Optimizer (from Pretrain)",
(
("muon_from_pretrain_pretrained", "### Pretrained VLM"),
("muon_from_pretrain_sft", "### VLM with SFT"),
),
),
(
"## VLM with Muon Optimizer (from SFT)",
(
("muon_from_sft_pretrained", "### Pretrained VLM"),
("muon_from_sft_sft", "### VLM with SFT"),
),
),
)
LABELS = tuple("ABCDEFGH")
METRICS = ("grounding_accuracy", "hallucination_control", "coverage", "visual_specificity")
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def canonical_json_bytes(value: Any) -> bytes:
return (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode()
def write_json(path: Path, value: Any) -> None:
path.write_bytes(canonical_json_bytes(value))
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def _parse_code_block(block: str, config: str) -> dict[str, Any]:
fence = re.search(r"```[^\n]*\n(.*?)\n```", block, flags=re.DOTALL)
if not fence:
raise ValueError(f"{config}: no evaluation code block")
transcript = fence.group(1).strip()
matches = list(
re.finditer(
r"(?m)^\[Image\]:\s*([^\n]+)\n🤖️:\s*(.*?)(?=\n(?:\s*\n)*\[Image\]:|\Z)",
transcript,
flags=re.DOTALL,
)
)
outputs = []
for match in matches:
image = match.group(1).strip()
output = match.group(2).strip()
if image not in IMAGE_FILES:
raise ValueError(f"{config}: unexpected image {image!r}")
if not output:
raise ValueError(f"{config}/{image}: empty output")
outputs.append({"image": image, "output": output})
if len(outputs) != len(IMAGE_FILES) or {row["image"] for row in outputs} != set(IMAGE_FILES):
raise ValueError(f"{config}: expected all eight images, found {len(outputs)}")
command = next(
(line.strip() for line in transcript.splitlines() if line.strip().startswith("$")), ""
)
return {
"config": config,
**CONFIG_META[config],
"historical_command": command,
"output_count": len(outputs),
"outputs": outputs,
}
def parse_retained_outputs(report_path: Path = REPORT_PATH) -> dict[str, Any]:
text = report_path.read_text(encoding="utf-8")
start = text.index("# Analysis of Vision-Language Model Training Results")
end = text.index("## Key Findings and Summary of VLM Training", start)
vlm = text[start:end]
cells = []
for section_index, (section_heading, stages) in enumerate(SECTION_SPECS):
section_start = vlm.index(section_heading)
section_end = (
vlm.index(SECTION_SPECS[section_index + 1][0], section_start)
if section_index + 1 < len(SECTION_SPECS)
else len(vlm)
)
section = vlm[section_start:section_end]
for stage_index, (config, heading) in enumerate(stages):
cell_start = section.index(heading)
cell_end = (
section.index(stages[stage_index + 1][1], cell_start)
if stage_index + 1 < len(stages)
else len(section)
)
cells.append(_parse_code_block(section[cell_start:cell_end], config))
if tuple(cell["config"] for cell in cells) != CONFIGS:
raise ValueError("historical VLM cells are incomplete or out of order")
return {
"schema_version": "exp8-4-retained-outputs-v1",
"experiment": "8-4",
"source_report": str(report_path.relative_to(REPO_ROOT)),
"source_report_sha256": sha256_file(report_path),
"cell_count": len(cells),
"output_count": sum(cell["output_count"] for cell in cells),
"images": list(IMAGE_FILES),
"configs": list(CONFIGS),
"cells": cells,
}
def outputs_for_image(retained: dict[str, Any], image: str) -> dict[str, str]:
rows = {}
for cell in retained["cells"]:
match = [row for row in cell["outputs"] if row["image"] == image]
if len(match) != 1:
raise ValueError(f"{cell['config']}/{image}: expected one retained output")
rows[cell["config"]] = match[0]["output"]
return rows
def blind_mapping(image: str) -> dict[str, str]:
configs = list(CONFIGS)
image_seed = int(hashlib.sha256(image.encode()).hexdigest()[:8], 16)
random.Random(BLIND_SEED + image_seed).shuffle(configs)
return dict(zip(LABELS, configs, strict=True))
def image_path(source_dir: Path, image: str) -> Path:
return source_dir / "dataset" / "eval_images" / IMAGE_FILES[image]
def image_data_url(path: Path) -> str:
mime = mimetypes.guess_type(path.name)[0] or "image/jpeg"
return f"data:{mime};base64,{base64.b64encode(path.read_bytes()).decode()}"
def judge_payload(
retained: dict[str, Any], image: str, source_dir: Path, model: str
) -> tuple[dict[str, Any], dict[str, str]]:
mapping = blind_mapping(image)
outputs = outputs_for_image(retained, image)
candidates = {label: outputs[config] for label, config in mapping.items()}
required = {
"image": image,
"candidates": {
label: {
**{metric: "number 0-5" for metric in METRICS},
"material_errors": ["specific image-grounding errors; empty only if none"],
"rationale": "brief evidence-based explanation",
}
for label in LABELS
},
"rank_order": list(LABELS),
"best": "one label A-H",
}
text = json.dumps(
{
"image": image,
"task": "Judge eight anonymous captions against the attached image.",
"rubric": {
"grounding_accuracy": "0 unrelated or false; 3 main subject mostly right; 5 all material claims visibly supported",
"hallucination_control": "0 dominated by invented objects/relations; 3 some speculation; 5 no material invention",
"coverage": "0 misses the scene; 3 covers main subject; 5 covers the important visible scene without padding",
"visual_specificity": "0 generic/nonvisual; 3 some concrete details; 5 precise discriminative visible details",
},
"candidates": candidates,
"required_json_shape": required,
},
ensure_ascii=False,
sort_keys=True,
)
payload = {
"model": model,
"temperature": 0,
"response_format": {"type": "json_object"},
"messages": [
{
"role": "system",
"content": (
"You are an independent vision-language evaluator. The candidates are anonymous. "
"Do not infer model identity, optimizer, base checkpoint, or training stage. Inspect "
"the attached image, score only visible grounding, and return exactly one JSON object."
),
},
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": image_data_url(image_path(source_dir, image)),
"detail": "high",
},
},
{"type": "text", "text": text},
],
},
],
}
return payload, mapping
def extract_json_object(content: str) -> dict[str, Any]:
stripped = content.strip()
if stripped.startswith("```"):
stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
stripped = re.sub(r"\s*```$", "", stripped)
try:
parsed = json.loads(stripped)
except json.JSONDecodeError:
start = stripped.find("{")
if start < 0:
raise
parsed, _ = json.JSONDecoder().raw_decode(stripped[start:])
if not isinstance(parsed, dict):
raise TypeError("judge content must decode to an object")
return parsed
def validate_judgment(judgment: dict[str, Any], image: str) -> None:
if judgment.get("image") != image:
raise ValueError(f"judge returned wrong image: {judgment.get('image')!r}")
candidates = judgment.get("candidates")
if not isinstance(candidates, dict) or set(candidates) != set(LABELS):
raise ValueError("judge must score A-H exactly")
for label in LABELS:
row = candidates[label]
if not isinstance(row, dict):
raise TypeError(f"candidate {label} score must be an object")
for metric in METRICS:
value = row.get(metric)
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not 0 <= value <= 5
):
raise ValueError(f"candidate {label} invalid {metric}: {value!r}")
if not isinstance(row.get("material_errors"), list):
raise TypeError(f"candidate {label} material_errors must be a list")
if not isinstance(row.get("rationale"), str) or not row["rationale"].strip():
raise ValueError(f"candidate {label} rationale is missing")
rank_order = judgment.get("rank_order")
if (
not isinstance(rank_order, list)
or len(rank_order) != len(LABELS)
or set(rank_order) != set(LABELS)
):
raise ValueError("rank_order must be a permutation of A-H")
if judgment.get("best") not in LABELS:
raise ValueError("best must be one label A-H")
def call_judge(
retained: dict[str, Any],
image: str,
*,
source_dir: Path,
endpoint: str,
model: str,
api_key: str,
timeout: float,
) -> dict[str, Any]:
payload, mapping = judge_payload(retained, image, source_dir, model)
request = urllib.request.Request(
endpoint,
data=json.dumps(payload, ensure_ascii=False).encode(),
method="POST",
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
)
started = time.perf_counter()
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
raw_body = response.read()
http_status = response.status
except urllib.error.HTTPError as exc:
body = exc.read().decode(errors="replace")
raise RuntimeError(f"judge HTTP {exc.code}: {body[:500]}") from exc
latency_ms = round((time.perf_counter() - started) * 1000, 3)
raw_response = json.loads(raw_body)
judgment = extract_json_object(raw_response["choices"][0]["message"]["content"])
validate_judgment(judgment, image)
response_id = raw_response.get("id")
usage = raw_response.get("usage")
if not isinstance(response_id, str) or not response_id:
raise ValueError("judge response has no response ID")
if not isinstance(usage, dict) or not isinstance(usage.get("total_tokens"), int):
raise TypeError("judge response has no complete usage")
path = image_path(source_dir, image)
return {
"image": image,
"image_source_filename": IMAGE_FILES[image],
"image_sha256": sha256_file(path),
"image_bytes": path.stat().st_size,
"provider": "ark",
"endpoint": endpoint,
"credential_env": "ARK_API_KEY",
"credential_headers_retained": False,
"blind_seed": BLIND_SEED,
"blind_map": mapping,
"request": payload,
"http_status": http_status,
"response": raw_response,
"response_id": response_id,
"usage": usage,
"latency_ms": latency_ms,
"judgment": judgment,
}
def reproduction_contract() -> dict[str, Any]:
return {
"schema_version": "exp8-4-reproduction-contract-v1",
"experiment": "8-4",
"historical_evidence_boundary": {
"historical_training_executed": True,
"eight_historical_cells_and_64_outputs_retained": True,
"historical_source_revisions_retained": False,
"historical_dataset_hashes_retained": False,
"historical_base_checkpoint_hashes_retained": False,
"historical_vlm_checkpoint_hashes_retained": False,
"historical_rng_and_stepwise_logs_retained": False,
"claim": (
"The author's report establishes that eight VLM configurations were evaluated on eight images. "
"It does not establish byte identity of the historical code, datasets, base/VLM checkpoints, or RNG state."
),
},
"future_reproduction": {
"vlm_source": {
"repository": "bojieli/minimind-v",
"original_revision": ORIGINAL_VLM_REVISION,
"original_files_sha256": ORIGINAL_VLM_FILES,
"qk_norm_muon_revision": IMPROVED_VLM_REVISION,
"qk_norm_muon_files_sha256": IMPROVED_VLM_FILES,
"not_claimed_as_historical_revisions": True,
},
"base_llm_source": {
"repository": "bojieli/minimind",
"original_revision": ORIGINAL_LLM_REVISION,
"original_files_sha256": ORIGINAL_LLM_FILES,
"qk_norm_muon_revision": IMPROVED_LLM_REVISION,
"qk_norm_muon_files_sha256": IMPROVED_LLM_FILES,
"dependency": "Use the Experiment 8-3 data/commands to produce original-SFT and improved pretrain/SFT/DPO 768-dimension base checkpoints.",
},
"vlm_dataset": {
"repository": "jingyaogong/minimind-v_dataset",
"revision": DATASET_REVISION,
"selected_for_jsonl_script_compatibility": True,
"files": DATASET_FILES,
},
"vision_encoder": {
"repository": "openai/clip-vit-base-patch16",
"revision": CLIP_REVISION,
"file": CLIP_FILE,
},
"evaluation_images": {
image: {"source_filename": IMAGE_FILES[image], "sha256": IMAGE_SHA256[image]}
for image in IMAGE_FILES
},
"commands": {
"original_source": "git clone https://github.com/bojieli/minimind-v.git sources/original-minimind-v && git -C sources/original-minimind-v checkout --detach 765908051d0837d60cecfb93f8390334e2e55f1e",
"improved_source": "git clone https://github.com/bojieli/minimind-v.git sources/qk-norm-muon-minimind-v && git -C sources/qk-norm-muon-minimind-v checkout --detach ead791c530fa5f9a3549dbfe9e11ec732d18d2e5",
"dataset": "git clone https://huggingface.co/datasets/jingyaogong/minimind-v_dataset dataset-source && git -C dataset-source checkout --detach ac9d03a3fd26a2d8e74bda374d9a2ddba49e4c1b && cp dataset-source/{pretrain_data.jsonl,sft_data.jsonl} dataset/ && unzip dataset-source/pretrain_images.zip -d dataset && unzip dataset-source/sft_images.zip -d dataset",
"vision_encoder": "git clone https://huggingface.co/openai/clip-vit-base-patch16 model/vision_model/clip-vit-base-patch16 && git -C model/vision_model/clip-vit-base-patch16 checkout --detach 57c216476eefef5ab752ec549e440a49ae4ae5f3",
"original_pretrain_vlm": "install -m 0644 <exp8-3-original-sft-768.pth> runs/original/out/llm_768.pth && cd trainer && torchrun --nproc_per_node=8 train_pretrain_vlm.py --out_dir ../runs/original/out --epochs 4 --hidden_size 768 --num_hidden_layers 16 --data_path ../dataset/pretrain_data.jsonl --images_path ../dataset/pretrain_images --use_wandb",
"original_sft_vlm": "cd trainer && torchrun --nproc_per_node=8 train_sft_vlm.py --out_dir ../runs/original/out --epochs 4 --hidden_size 768 --num_hidden_layers 16 --data_path ../dataset/sft_data.jsonl --images_path ../dataset/sft_images --use_wandb",
"improved_matrix": "For each BASE in pretrain,sft,dpo, install the corresponding Experiment-8-3 QK-Norm+Muon 768-dimension checkpoint as runs/muon-from-$BASE/out/llm_768.pth, then run train_pretrain_vlm_muon.py and train_sft_vlm_muon.py with the same four-epoch data arguments in that isolated out_dir.",
"evaluation": "For every isolated out_dir, preserve both checkpoints, copy the selected *_muon_768.pth name to eval_vlm.py's pretrain_vlm_768.pth or sft_vlm_768.pth compatibility name when needed, then run python eval_vlm.py --load 0 --model_mode 0 and --model_mode 1 on the eight hash-pinned images with seed 1337.",
},
"environment": {
"book_pyproject": "pyproject.toml",
"book_lock": "uv.lock",
"install": "uv sync --locked --python 3.12 --extra ch7 --extra dev",
"boundary": "The book lock freezes a future Python environment; CUDA, drivers, and the historical GPU image were not retained.",
},
},
"reported_training_design": {
"parameter_count_millions": {"original": 104.622, "qk_norm_muon": 104.625},
"base_llm_stages": ["pretrain", "sft", "dpo"],
"vlm_stages": ["pretrained", "sft"],
"projection_pretraining_freezes_llm": True,
"sft_unfreezes_full_model": True,
"reported_epochs": 4,
"seed_in_current_source": 1337,
"source_verified_mechanisms": {
"original_revision_precedes_qk_norm_commit": True,
"improved_revision_has_qk_norm_before_rope": True,
"improved_revision_uses_muon_for_selected_2d_weights": True,
"vision_encoder_is_frozen_clip": True,
},
},
"checkpoint_policy": {
"distributed_with_book": False,
"acceptance_artifact": False,
"required_artifact": "reproducible evidence-backed training report",
"reason": "Training checkpoints are intentionally not distributed to readers.",
},
}
def mean(values: list[float]) -> float:
return round(sum(values) / len(values), 4)
def summarize(
retained: dict[str, Any], receipts: list[dict[str, Any]], contract: dict[str, Any]
) -> dict[str, Any]:
per_image_config_scores: dict[str, dict[str, Any]] = {}
best_counts = {config: 0 for config in CONFIGS}
for receipt in receipts:
scores = {
receipt["blind_map"][label]: row
for label, row in receipt["judgment"]["candidates"].items()
}
per_image_config_scores[receipt["image"]] = scores
best_counts[receipt["blind_map"][receipt["judgment"]["best"]]] += 1
config_averages = {}
for config in CONFIGS:
rows = [per_image_config_scores[image][config] for image in IMAGE_FILES]
config_averages[config] = {
metric: mean([float(row[metric]) for row in rows]) for metric in METRICS
}
config_averages[config]["overall"] = mean(
[float(row[metric]) for row in rows for metric in METRICS]
)
stage_averages = {}
for stage in ("pretrained", "sft"):
configs = [config for config in CONFIGS if CONFIG_META[config]["vlm_stage"] == stage]
stage_averages[stage] = {
metric: mean([config_averages[config][metric] for config in configs])
for metric in (*METRICS, "overall")
}
isolated_pairs = {}
for stage in ("pretrained", "sft"):
original = f"without_muon_{stage}"
improved = f"muon_from_sft_{stage}"
isolated_pairs[stage] = {
"original": config_averages[original]["overall"],
"qk_norm_muon_from_sft": config_averages[improved]["overall"],
"delta": round(
config_averages[improved]["overall"] - config_averages[original]["overall"], 4
),
}
response_ids = [receipt["response_id"] for receipt in receipts]
acceptance = {
"historical_report_content_hashed": bool(retained["source_report_sha256"]),
"all_eight_configuration_cells_retained": retained["cell_count"] == 8,
"all_64_historical_outputs_retained": retained["output_count"] == 64,
"same_eight_images_present_in_every_cell": all(
cell["output_count"] == 8 for cell in retained["cells"]
),
"eight_image_aware_arm_blind_judgments": len(receipts) == 8,
"raw_judge_requests_responses_ids_usage_latency_retained": len(set(response_ids)) == 8
and all(
receipt["usage"].get("total_tokens", 0) > 0 and receipt["latency_ms"] > 0
for receipt in receipts
),
"request_images_match_pinned_sha256": all(
receipt["image_sha256"] == IMAGE_SHA256[receipt["image"]] for receipt in receipts
),
"immutable_original_and_improved_source_revisions_frozen": bool(
ORIGINAL_VLM_REVISION and IMPROVED_VLM_REVISION
),
"immutable_dataset_clip_and_eval_image_inputs_frozen": bool(
DATASET_REVISION and CLIP_REVISION and IMAGE_SHA256
),
"future_reproduction_commands_declared": len(contract["future_reproduction"]["commands"])
>= 6,
"historical_provenance_limitations_explicit": contract["historical_evidence_boundary"][
"historical_vlm_checkpoint_hashes_retained"
]
is False,
"checkpoints_not_an_acceptance_artifact": contract["checkpoint_policy"][
"acceptance_artifact"
]
is False,
}
passed = all(acceptance.values())
ranking = sorted(CONFIGS, key=lambda config: (-config_averages[config]["overall"], config))
return {
"schema_version": "exp8-4-summary-v1",
"experiment": "8-4",
"status": "passed" if passed else "failed",
"judge": {
"provider": "ark",
"model": receipts[0]["request"]["model"],
"image_aware": True,
"calls": len(receipts),
"response_ids": response_ids,
"total_tokens": sum(receipt["usage"]["total_tokens"] for receipt in receipts),
"total_latency_ms": round(sum(receipt["latency_ms"] for receipt in receipts), 3),
"blind_seed": BLIND_SEED,
},
"retained": {
"cells": retained["cell_count"],
"outputs": retained["output_count"],
"images": len(retained["images"]),
},
"config_averages": config_averages,
"stage_averages": stage_averages,
"isolated_original_vs_qk_norm_muon_from_sft": isolated_pairs,
"ranking_by_overall": ranking,
"best_counts": best_counts,
"per_image_config_scores": per_image_config_scores,
"scientific_findings": {
"top_configuration": ranking[0],
"top_configuration_overall": config_averages[ranking[0]]["overall"],
"sft_minus_pretrained_average": round(
stage_averages["sft"]["overall"] - stage_averages["pretrained"]["overall"], 4
),
"author_claims_are_historical_observations_not_acceptance_gates": True,
"muon_only_causal_claim_avoided": True,
},
"acceptance": {**acceptance, "passed": passed},
"limitations": [
"Historical base-LLM and VLM checkpoints are intentionally not distributed and were not recreated in this audit.",
"Historical source revisions, dataset identities, RNG state, hardware image, and stepwise logs were not retained.",
"Current immutable pins define a future reproduction contract and are not represented as the exact historical run.",
"The English captions are translations in a bilingual report, so translation can affect judging.",
"One image-aware judge call evaluates all eight anonymous candidates per image; scores are descriptive, not a powered significance test.",
"QK-Norm and Muon change together in the improved arm, so the report does not attribute effects to Muon alone.",
],
}
def render_report(summary: dict[str, Any]) -> str:
lines = [
"# Experiment 8-4 retained-training-report audit",
"",
"## Result",
"",
f"Status: **{summary['status']}**. The historical report retains {summary['retained']['outputs']} image descriptions across {summary['retained']['cells']} configurations and the same {summary['retained']['images']} images. Each image was inspected by a real image-capable ARK judge together with all eight arm-blind captions.",
"",
"| Configuration | Grounding | Hallucination control | Coverage | Specificity | Overall | Best count |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for config in summary["ranking_by_overall"]:
row = summary["config_averages"][config]
lines.append(
f"| {config} | {row['grounding_accuracy']:.4f} | {row['hallucination_control']:.4f} | {row['coverage']:.4f} | {row['visual_specificity']:.4f} | {row['overall']:.4f} | {summary['best_counts'][config]} |"
)
findings = summary["scientific_findings"]
lines.extend(
[
"",
f"The highest descriptive judge mean was **{findings['top_configuration']}** at **{findings['top_configuration_overall']:.4f}**. Averaged across all four base configurations, full VLM SFT changed the score by **{findings['sft_minus_pretrained_average']:+.4f}** versus projection-only pretraining.",
"",
"The isolated report comparison pairs original/SFT-base against QK-Norm+Muon/SFT-base at each VLM stage. QK-Norm and Muon still change together, so no Muon-only causal claim is made. All author-written qualitative claims remain historical observations rather than pass/fail gates.",
"",
"## Provenance and reproduction boundary",
"",
"`reproduction_contract.json` freezes separate pre-QK-Norm and QK-Norm+Muon MiniMind-V revisions, the corresponding base-LLM revisions, script-compatible VLM dataset Git-LFS objects, the CLIP weight object, all eight evaluation-image hashes, and future commands. These pins are not misrepresented as the historical checkout.",
"",
"Training checkpoints remain local by book policy and are not acceptance artifacts. The accepted artifact is this content-hashed report, all 64 retained outputs, eight raw image-aware judge receipts, and explicit limitations.",
"",
]
)
return "\n".join(lines)
def input_record(path: Path) -> dict[str, Any]:
return {
"path": str(path.relative_to(REPO_ROOT)),
"sha256": sha256_file(path),
"bytes": path.stat().st_size,
}
def artifact_record(path: Path, run_dir: Path) -> dict[str, Any]:
return {
"path": str(path.relative_to(run_dir)),
"sha256": sha256_file(path),
"bytes": path.stat().st_size,
}
def build_manifest(run_id: str, run_dir: Path, summary: dict[str, Any]) -> dict[str, Any]:
inputs = [
input_record(REPORT_PATH),
input_record(REPO_ROOT / "pyproject.toml"),
input_record(REPO_ROOT / "uv.lock"),
input_record(HERE / "run_vlm_training_report_audit.py"),
input_record(HERE / "validate_vlm_evidence.py"),
input_record(HERE / "test_vlm_training_report_audit.py"),
]
artifacts = [
run_dir / name
for name in (
"retained_outputs.json",
"reproduction_contract.json",
"judge_receipts.json",
"summary.json",
"report.md",
)
]
return {
"schema_version": "exp8-4-manifest-v1",
"experiment": "8-4",
"run_id": run_id,
"created_at": utc_now(),
"status": summary["status"],
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
"inputs": inputs,
"artifacts": [artifact_record(path, run_dir) for path in artifacts],
"acceptance": summary["acceptance"],
"checkpoint_policy": "not distributed; not an acceptance artifact",
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--run-id", default=DEFAULT_RUN_ID)
parser.add_argument("--source-dir", type=Path, default=os.getenv("MINIMIND_V_SOURCE_DIR"))
parser.add_argument("--endpoint", default=os.getenv("ARK_BASE_URL", DEFAULT_ENDPOINT))
parser.add_argument("--model", default=os.getenv("ARK_VISION_MODEL", DEFAULT_MODEL))
parser.add_argument("--api-key-env", default="ARK_API_KEY")
parser.add_argument("--timeout", type=float, default=240.0)
parser.add_argument("--concurrency", type=int, default=4)
parser.add_argument(
"--refresh-manifest",
action="store_true",
help="Rehash an existing run without provider calls; refuses changed retained outputs.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
if not re.fullmatch(r"[A-Za-z0-9._-]+", args.run_id):
raise SystemExit("run ID may contain only letters, digits, dot, underscore, and hyphen")
run_dir = RUNS_DIR / args.run_id
if args.refresh_manifest:
if not run_dir.is_dir():
raise SystemExit(f"cannot refresh missing run: {run_dir}")
stored = json.loads((run_dir / "retained_outputs.json").read_text(encoding="utf-8"))
retained = parse_retained_outputs()
if stored.get("cells") != retained.get("cells"):
raise SystemExit(
"refusing manifest refresh because retained historical outputs changed"
)
receipts_doc = json.loads((run_dir / "judge_receipts.json").read_text(encoding="utf-8"))
receipts = receipts_doc.get("calls")
if (
receipts_doc.get("schema_version") != "exp8-4-judge-receipts-v1"
or receipts_doc.get("experiment") != "8-4"
or not isinstance(receipts, list)
):
raise SystemExit("cannot refresh malformed judge receipts")
contract = reproduction_contract()
summary = summarize(retained, receipts, contract)
write_json(run_dir / "retained_outputs.json", retained)
write_json(run_dir / "reproduction_contract.json", contract)
write_json(run_dir / "summary.json", summary)
(run_dir / "report.md").write_text(render_report(summary), encoding="utf-8")
write_json(run_dir / "manifest.json", build_manifest(args.run_id, run_dir, summary))
latest = {
"schema_version": "exp8-4-latest-v1",
"experiment": "8-4",
"run_id": args.run_id,
"status": summary["status"],
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
"manifest_sha256": sha256_file(run_dir / "manifest.json"),
}
write_json(LATEST_PATH, latest)
print(json.dumps(latest, indent=2, sort_keys=True))
return 0
if run_dir.exists():
raise SystemExit(f"refusing to overwrite existing run: {run_dir}")
if args.source_dir is None:
raise SystemExit("--source-dir or MINIMIND_V_SOURCE_DIR is required")
source_dir = args.source_dir.resolve()
for image, expected in IMAGE_SHA256.items():
path = image_path(source_dir, image)
if not path.is_file() or sha256_file(path) != expected:
raise SystemExit(f"evaluation image missing or hash mismatch: {path}")
api_key = os.getenv(args.api_key_env)
if not api_key:
raise SystemExit(f"missing required credential environment variable: {args.api_key_env}")
retained = parse_retained_outputs()
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futures = {
image: pool.submit(
call_judge,
retained,
image,
source_dir=source_dir,
endpoint=args.endpoint,
model=args.model,
api_key=api_key,
timeout=args.timeout,
)
for image in IMAGE_FILES
}
receipts = [futures[image].result() for image in IMAGE_FILES]
contract = reproduction_contract()
summary = summarize(retained, receipts, contract)
run_dir.mkdir(parents=True)
write_json(run_dir / "retained_outputs.json", retained)
write_json(
run_dir / "judge_receipts.json",
{
"schema_version": "exp8-4-judge-receipts-v1",
"experiment": "8-4",
"credential_headers_retained": False,
"calls": receipts,
},
)
write_json(run_dir / "reproduction_contract.json", contract)
write_json(run_dir / "summary.json", summary)
(run_dir / "report.md").write_text(render_report(summary), encoding="utf-8")
write_json(run_dir / "manifest.json", build_manifest(args.run_id, run_dir, summary))
latest = {
"schema_version": "exp8-4-latest-v1",
"experiment": "8-4",
"run_id": args.run_id,
"status": summary["status"],
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
"manifest_sha256": sha256_file(run_dir / "manifest.json"),
}
write_json(LATEST_PATH, latest)
print(json.dumps(latest, indent=2, sort_keys=True))
return 0 if summary["status"] == "passed" else 1
if __name__ == "__main__":
raise SystemExit(main())
chapter8/MiniMind-pretrain/validation/runs/exp8-3-training-report-20260731-v1/judge_receipts.json
+804
View File
File diff suppressed because one or more lines are too long
chapter8/MiniMind-pretrain/validation/runs/exp8-3-training-report-20260731-v1/manifest.json
+79
View File
@@ -0,0 +1,79 @@
{
"acceptance": {
"all_expected_outputs_retained": true,
"all_six_arm_stage_cells_retained": true,
"checkpoints_not_an_acceptance_artifact": true,
"eight_stage_balanced_blind_judgments": true,
"future_reproduction_commands_declared": true,
"historical_provenance_limitations_explicit": true,
"immutable_dataset_revision_lfs_hashes_and_sizes_frozen": true,
"immutable_source_revision_and_file_hashes_frozen": true,
"original_and_qk_norm_muon_compared": true,
"passed": true,
"pretrain_sft_and_dpo_compared": true,
"raw_historical_report_hashed": true,
"raw_judge_requests_responses_ids_usage_latency_retained": true,
"reported_loss_claims_qualified": true
},
"artifacts": [
{
"bytes": 41195,
"path": "retained_outputs.json",
"sha256": "c38ba7fcf766e15e43712ceb7d99ccadcbea4400caf18d28993d7a3f7c6dc88a"
},
{
"bytes": 5555,
"path": "reproduction_contract.json",
"sha256": "224df3576cf8751ef1a1ceb6d879db0afec1347a79c094a02c372982aa3068e3"
},
{
"bytes": 90942,
"path": "judge_receipts.json",
"sha256": "e254be18e34aab27812d529c330490956545be3b66b5b59eb0958fb3c45e8e5b"
},
{
"bytes": 12556,
"path": "summary.json",
"sha256": "d06d354c8ed8e2d84c506b96dd7bdea0a6cfff36be24b2d731d5168855228740"
},
{
"bytes": 1464,
"path": "report.md",
"sha256": "a62514227ffb7770eb84dbe5f965df7f215582b8b481d13a7beebf9a7a4b28dd"
}
],
"checkpoint_policy": "not distributed; not an acceptance artifact",
"created_at": "2026-08-17T05:37:04.936375+00:00",
"experiment": "8-3",
"inputs": [
{
"bytes": 155255,
"path": "chapter8/MiniMind-pretrain/README.md",
"sha256": "a8e1df1a9ee5cf013995e9ff3b963621485a838c97456b9f30e65ea9fdf55d50"
},
{
"bytes": 14150,
"path": "pyproject.toml",
"sha256": "33cc27c2759f353663d6907f1a918a41a49fb8d3e4e28370d508136449c3b156"
},
{
"bytes": 1311181,
"path": "uv.lock",
"sha256": "347fc87f40526372c284c61e5374536ce8d2071936cef8844d350470ecdf0d0b"
},
{
"bytes": 34784,
"path": "chapter8/MiniMind-pretrain/validation/run_training_report_audit.py",
"sha256": "b2d43301a633cac2fa1fb7d29bfa831837b8dc7b27b2f887a0b70e45d68570f1"
},
{
"bytes": 14855,
"path": "chapter8/MiniMind-pretrain/validation/validate_evidence.py",
"sha256": "cc49d5185fae5680a5aac4d27b831b8a7215f3d7339ac46a998812e81bc13c9b"
}
],
"run_dir": "validation/runs/exp8-3-training-report-20260731-v1",
"run_id": "exp8-3-training-report-20260731-v1",
"schema_version": "exp8-3-manifest-v1",
"status": "passed"
}
chapter8/MiniMind-pretrain/validation/runs/exp8-3-training-report-20260731-v1/report.md
+20
View File
@@ -0,0 +1,20 @@
# Experiment 8-3 retained-training-report audit
## Result
Status: **passed**. The historical report retains 49 outputs across the original and QK-Norm + Muon arms after pretrain, SFT, and DPO. Eight preregistered arm-blind comparisons were judged from raw retained text by an independent ARK model.
| Arm | Fluency | Instruction | Factuality | Overall |
| --- | ---: | ---: | ---: | ---: |
| Original | 3.0000 | 1.7500 | 1.3750 | 2.0417 |
| QK-Norm + Muon | 3.7500 | 3.0000 | 4.1250 | 3.6250 |
Observed blind-judge overall delta: **+1.5833**. Pairwise decisions: {'original': 0, 'qk_norm_muon': 7, 'tie': 1}.
The report's loss claims (3.0 reached at 36 versus 12 reported steps; final loss 2.0 versus 1.7) are retained as author-reported observations, not independently recomputed measurements, because the historical stepwise logs were not preserved.
## Provenance and reproduction boundary
`reproduction_contract.json` freezes the MiniMind source revision, hashes the relevant source files, freezes a dataset revision with the three Git-LFS object hashes and sizes, and records all six future reproduction commands. These pins were selected for future reproduction and are not represented as the exact historical checkout.
Training checkpoints remain local by book policy and are not an acceptance artifact. The accepted artifact is this content-hashed training report, its raw retained outputs, raw independent-judge receipts, and explicit limitations.
chapter8/MiniMind-pretrain/validation/runs/exp8-3-training-report-20260731-v1/reproduction_contract.json
+106
View File
@@ -0,0 +1,106 @@
{
"checkpoint_policy": {
"acceptance_artifact": false,
"distributed_with_book": false,
"reason": "Training checkpoints are intentionally not distributed to readers.",
"required_artifact": "reproducible evidence-backed training report"
},
"experiment": "8-3",
"future_reproduction": {
"commands": {
"improved_dpo": "torchrun --nproc_per_node=8 trainer/train_dpo.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --init_from out/full_sft_muon_768.pth --data_path dataset/dpo.jsonl --use_wandb",
"improved_pretrain": "torchrun --nproc_per_node=8 trainer/train_pretrain_muon.py --epochs 10 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/pretrain_hq.jsonl --use_wandb",
"improved_sft": "torchrun --nproc_per_node=8 trainer/train_full_sft_muon.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/sft_512.jsonl --use_wandb",
"original_dpo": "torchrun --nproc_per_node=8 trainer/train_dpo.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --init_from out/full_sft_768.pth --data_path dataset/dpo.jsonl --use_wandb",
"original_pretrain": "torchrun --nproc_per_node=8 trainer/train_pretrain.py --epochs 10 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/pretrain_hq.jsonl --use_wandb",
"original_sft": "torchrun --nproc_per_node=8 trainer/train_full_sft.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/sft_512.jsonl --use_wandb"
},
"dataset": {
"files": {
"dpo.jsonl": {
"bytes": 53653322,
"lfs_sha256": "ee934a8a455ccc99d1334d63e1254dd1d64f497fd067cfcbb71e3043f5b46768"
},
"pretrain_hq.jsonl": {
"bytes": 1669750047,
"lfs_sha256": "9801b0d2210c61c2e4bc130f6dc4b3c870698a88d04af8f103c23dd5f0ce2440"
},
"sft_512.jsonl": {
"bytes": 7531517862,
"lfs_sha256": "053b7d09574e48a86232e929211434ff9e5016c6ed13312e63687dd52edcbebf"
}
},
"not_claimed_as_historical_revision": true,
"repository": "jingyaogong/minimind_dataset",
"revision": "84983ed4dec7836d240577760c1d6be5d4cabcf9",
"selected_at": "2026-07-31"
},
"environment": {
"book_lock": "uv.lock",
"book_pyproject": "pyproject.toml",
"boundary": "The book lock freezes a future software environment. The pinned upstream requirements file itself is unversioned, and the future GPU/CUDA stack has not been exercised here.",
"install": "uv sync --locked --python 3.12 --extra ch7 --extra dev"
},
"source": {
"file_sha256": {
"dataset/lm_dataset.py": "213726b1781289548784220b2b2db48fe97d84f3b67ac0cd70186cbbcb7b5d2c",
"eval_model.py": "43930a4b55048a4a3ffa17eb78ae67d59582d639aa9365f0bbf41ba149128af8",
"model/model_minimind.py": "2d33988711c704be6a22c4c61489b23106a2340a7cb8b97ebe3e40f30819cbb0",
"model/tokenizer.json": "e489029175fb3f94b8211a120a72a2ee41a664db65b828d077c7bde989c845a9",
"model/tokenizer_config.json": "190cc4738bac3b6f6b563376019c581b320fdb0260a03b9d5ab806296c8c6bb8",
"requirements.txt": "23f4cea09281765eec7cf03e28231425638e8e42580f418781fed166b75af968",
"trainer/muon.py": "00c2c6a225edeb55433df0724c3c74f6ff98ac4b2cc73c4aafcff686824f6267",
"trainer/train_dpo.py": "97f2c31cc8bc21a777e2efcb5e2fa35a49e4e9e3698db120148f8a0b2f678449",
"trainer/train_full_sft.py": "a57422f1df80bf2867f31f3b4a646a92ac7f66729e98a3b32cd1ec4d6780cb8b",
"trainer/train_full_sft_muon.py": "acd0b7db5b1d8b25d3c3103f92d68a7d381f9322a1b33bbec34be3d005930bad",
"trainer/train_pretrain.py": "ddd122645a9f1043bc8dac69a81ac51d2df95df8745d25faed7963d38fedc328",
"trainer/train_pretrain_muon.py": "fc83d07754ec3a8c156b6b8bfc0fd4326edecb72efabc5e08ae4ff5e3a7029bc"
},
"not_claimed_as_historical_revision": true,
"repository": "bojieli/minimind",
"revision": "8bdc5d97d5845a8c1ac2ed56a5b8b4c0d0fb0795",
"selected_at": "2026-07-31"
}
},
"historical_evidence_boundary": {
"claim": "The author's retained report supports that original and QK-Norm+Muon 104.03M models were evaluated after pretrain, SFT, and DPO. It does not establish byte identity of the historical checkpoints, datasets, source checkout, or every loss point.",
"historical_checkpoint_hashes_retained": false,
"historical_dataset_hashes_retained": false,
"historical_source_revision_retained": false,
"historical_stepwise_training_logs_retained": false,
"historical_training_executed": true,
"six_historical_evaluation_transcripts_retained": true
},
"model_and_training": {
"architecture": {
"hidden_size": 768,
"layers": 16,
"sequence_length": 512
},
"arms": [
"original",
"qk_norm_muon"
],
"reported_parameter_count_millions": 104.03,
"reported_scalars_without_stepwise_logs": {
"eight_rtx_4090_pretrain_hours": 6,
"eight_rtx_4090_sft_hours": 8,
"final_loss_original": 2.0,
"final_loss_qk_norm_muon": 1.7,
"steps_to_loss_3_original": 36,
"steps_to_loss_3_qk_norm_muon": 12
},
"source_verified_mechanisms": {
"adamw_for_embeddings_norms_and_lm_head": true,
"dpo_uses_adamw_from_arm_specific_sft_checkpoint": true,
"muon_for_two_dimensional_non_embedding_weights": true,
"qk_norm_before_rope": true
},
"stages": [
"pretrain",
"sft",
"dpo"
]
},
"schema_version": "exp8-3-reproduction-contract-v1"
}
chapter8/MiniMind-pretrain/validation/runs/exp8-3-training-report-20260731-v1/retained_outputs.json
+263
View File
@@ -0,0 +1,263 @@
{
"arms": [
"original",
"qk_norm_muon"
],
"cell_count": 6,
"cells": [
{
"arm": "original",
"pair_count": 7,
"pairs": [
{
"output": "and the relationship of universal laws are the premise of the basic principles of Marxism. The basic principles of Marxism are the relationship of universal laws. Basic principles refer to those determined by universal laws, where universal laws are \"universal laws.\" Universal laws refer to those determined by universal laws, but they are not absolute laws; they are only a form of rules, a part of universal laws. The basic principles of Marxism include universal laws, that is, those determined by universal laws, rather than absolute laws.",
"prompt": "Basic principles of Marxism"
},
{
"output": "What are they? The main functions of the human brain include:\n1. Memory: The formation and retrieval of memory are through chemical and electrical signals between neurons in the brain, storing and retrieving information.\n2. Decision-making: Decision-making is the ultimate goal of the brain. It is formed by the connections and interactions between neurons and neurons.\n3. Senses: Perception, sensation, action, and feeling are all closely related to brain function. Senses are the sensory information in our brain, including vision, hearing, smell, taste, smell, and touch. Senses are the sensory information in our brain, including vision, hearing, smell, taste, smell, and touch.\nThese functions and structures together help us complete complex thinking and behavior.",
"prompt": "The main functions of the human brain"
},
{
"output": "to study the interaction between celestial bodies. It firmly places all objects on a flat surface, forming a torque, called universal gravitation. The law of universal gravitation is one of the three basic laws discovered by Newton. It is a scientific theory describing how matter in the universe interacts. The law of universal gravitation describes the interaction between objects, placing all objects firmly on a flat surface, forming a torque, called Newton's law. It is one of the three basic laws discovered by Newton, a scientific theory describing how matter in the universe interacts.",
"prompt": "The principle of universal gravitation is"
},
{
"output": "which one? The highest mountain in the world is Mount Everest, located in the Himalayas, with an altitude of 8,848 meters.\nMount Everest is one of the most challenging climbing targets on Earth, attracting climbers from all over the world every year. Climbing Mount Everest requires climbers to overcome various challenges such as gravity, altitude, and physical strength. At the same time, Mount Everest is also one of the world's natural wonders, known as the \"Roof of the World.\"\nMount Everest is located on the border between Nepal and China, in the Himalayas. In Nepali, its name means \"Goddess's Home,\" meaning \"Goddess's Home.\" In the Himalayas, there are many peaks, the highest of which is Mount Everest, with an altitude of 8,848 meters.",
"prompt": "The highest mountain in the world is"
},
{
"output": "has a concentration of zero, right? Wrong.\nCarbon dioxide is a gas with the molecular formula CO2. It is a molecule composed of one carbon atom and two oxygen atoms. Carbon dioxide is a very important element, widely present in nature. It is also one of the main gases for respiration and digestion in humans and other animals.\nAlthough the concentration of carbon dioxide in the air is high, this does not mean it is a harmful chemical. It only accounts for about 20% of the air. Other carbon compounds (such as glucose and starch) can also have a negative impact on human health.\nTherefore, carbon dioxide is not a harmful chemical.",
"prompt": "Carbon dioxide in the air"
},
{
"output": "7, what are their sizes? This number is relatively common, because there are many types of animals on Earth, but the answer is 7.\nIn fact, the largest animal on Earth is the blue whale, which can reach a size of over 100 feet (30 meters). The blue whale is a marine mammal, often considered a sea giant because they live in the ocean. The blue whale is currently the largest known animal, with a weight of up to 200 tons and a length of up to 30 meters.\nIn addition, there are many other different types of organisms in the ocean, such as plankton, mollusks, etc. The sizes and shapes of these organisms vary, some can even grow to over 300 feet, and some can even reach 40 feet.\nIn summary, although we cannot directly classify all animals on Earth into one category, we can be sure that there are many different types of animals on Earth.",
"prompt": "The largest animal on Earth has"
},
{
"output": "what? Hangzhou has a wide variety of specialty delicacies, among which the most famous might be West Lake Vinegar Fish, renowned for its Zhejiang cuisine. In addition, there are West Lake Vinegar Fish, Zhejiang Longjing Shrimp, Zhejiang Snail Noodles, etc. If you want to try other Hangzhou delicacies, you can also tell me, and I can provide you with more information.",
"prompt": "The delicacies of Hangzhou include"
}
],
"stage": "pretrain",
"terminal_header": "### Pretrain Model\n\n\n$ python eval_model.py --load 0 --model 0\nMiniMind model parameter count: 104.03M(illion)\n[0] Auto test\n[1] Manual input\n0"
},
{
"arm": "original",
"pair_count": 8,
"pairs": [
{
"output": "As a chatbot, my existence is mainly to provide users with useful information, helping them solve problems and obtain information. I can answer a variety of common questions, provide various information, including history, culture, geography, science, technology, society, culture, etc. I can also learn and understand human language, language",
"prompt": "Please introduce yourself."
},
{
"output": "As an AI assistant, I don't have the ability to \"like\" like a human. My purpose is to answer questions raised by users and provide useful information and help. I am just a program designed to answer various questions and provide useful information. If you have any other questions, please let me know, and I will try my best to answer.\n\nLu Xun's \"Diary of a Madman\" reveals the hypocrisy and emptiness of feudal ethics. The novel, through the protagonist's inner monologue, reveals the hypocrisy and emptiness existing in society at that time. This contradiction and emptiness also led people to question and reflect on feudal ethics.\n\nAt the same time, the novel also reveals the influence and change of feudal ethics on society and individuals through the depiction of the characters' life experiences and inner worlds. This critical literary style and theme also made Lu Xun's literary works have a profound impact on society.\n\nIn summary, \"Diary of a Madman\" is a representative work of Lu Xun in literary creation. It criticizes the hypocrisy and emptiness of feudal ethics by depicting social problems at that time, while also reflecting various social issues and contradictions of that time.",
"prompt": "Which subject are you better at?"
},
{
"output": "First, you may have a cold, but this depends on your constitution, age, gender, condition, and medical history. If your constitution is weak, you may experience symptoms such as coughing and sore throat. In this case, it is recommended that you go to the hospital for an examination to determine the exact cause. The doctor may recommend that you use antibiotics or cold medicine to treat your symptoms. Based on the examination results, you can consider taking antibiotics, over-the-counter medications, or other drugs to relieve symptoms. In addition, you can also consider taking over-the-counter medications to relieve symptoms, such as antibiotics. However, it is best to consult a doctor to determine if it is suitable for your situation.",
"prompt": "I have been coughing for two weeks. Do I need to go to the hospital for a check-up?"
},
{
"output": "The speed of light refers to the speed at which light propagates in a vacuum, approximately 299,792,458 meters per second. This is a common unit of speed in physics. The speed of light is the most fundamental constant in nature, and its definition is that the speed of all matter in a vacuum exceeds one million kilometers per second. The speed of light is the propagation speed between matter in the universe, defined as the square of the speed of light in a vacuum divided by the square of the speed of light, that is, $c^2=m^2$. The absolute value of the speed of light is 2997*2458 meters per second, which is the speed of matter in a vacuum, approximately 4.08 meters per second. The speed of light has different meanings in different situations. For example, in the spectrum of a light source, the propagation speed of light is affected by the intensity and wavelength of light\n, while in the spectrum, the propagation speed of light is affected by information such as the intensity, frequency, brightness, and wavelength of light. The speed of light is constant in macroscopic objects, approximately 299,792,458 meters per second. In physics, the speed of light is constant, but for the propagation speed of light from a source, it is limited to a tiny range, even\nunable to transmit. The speed of light is the highest point of the source speed propagating in a vacuum, while the limitation of the source speed refers to the magnitude of the source speed. This limitation is caused by the relative positional relationship between the energy density of the source and the speed of light.",
"prompt": "Explain the physical concept of the speed of light in detail."
},
{
"output": "Hangzhou has many specialty delicacies, among which the more famous ones include:\n1. West Lake Vinegar Fish: Also known as \"Jiangsu fish,\" it is one of the traditional famous dishes of Hangzhou, with delicious soup and fresh carp.\n2. Crayfish: A traditional snack represented by crayfish, it is one of the unique snacks of Hangzhou.\n3. Hangzhou Xiaolongbao: Mainly made with tender meat filling, delicious soup, and fresh vegetables.\n4. Longjing Shrimp: Represented by fresh shrimp, it has a delicious seafood flavor and is also a very famous specialty snack of Hangzhou.\n5. Xitang: Famous for its ancient town, mainly made with fresh fish, paired with fresh peanuts and taro and other snacks, it is a very famous snack of Hangzhou.\nI hope the above recommendations are helpful to you.",
"prompt": "Recommend some specialty delicacies of Hangzhou."
},
{
"output": "A large language model refers to a neural network-based language model, mainly used for natural language processing tasks. It can perform tasks such as text summarization, machine translation, and dialogue systems. It can process large amounts of language data and automatically extract meaningful vocabulary, syntactic structures, and other information from it, providing support for fields such as text generation and speech recognition.",
"prompt": "Please explain the concept of \"large language model\" to me."
},
{
"output": "ChatGPT is a natural language processing tool developed by Google, designed to help users create meaningful conversations to understand natural language. It uses speech recognition technology, by analyzing the user's input language, it can understand the user's input, and use semantic analysis technology to help users answer various questions. It also supports text\ngeneration, can generate text, allowing users to create their own answers to understand the user's intent.",
"prompt": "How to understand ChatGPT?"
},
{
"output": "Introduce the history of the United States embers.",
"prompt": "Introduce the history of the United States, please."
}
],
"stage": "sft",
"terminal_header": "### SFT Model\n\n\n$ python eval_model.py --load 0 --model 1\nMiniMind model parameter count: 104.03M(illion)\n[0] Auto test\n[1] Manual input\n0"
},
{
"arm": "original",
"pair_count": 9,
"pairs": [
{
"output": "Hello! I am an AI language model capable of natural language conversation, answering questions, and generating text. I am a language model trained on large amounts of data that can understand and generate natural language.",
"prompt": "Please introduce yourself."
},
{
"output": "I am better at mathematics and logical reasoning. Although mathematics and logical reasoning are relatively abstract subjects, there is no necessary connection between them. Mathematics is a tool used to describe and solve problems, while logical reasoning is the process of reasoning and proving inferences. Mathematics is a foundational subject, and mathematics is the foundational subject of logic. Mathematics is a discipline that studies the concepts of quantity and space. Logical reasoning is a common branch of mathematics, mainly used for reasoning and proving the process of inference. Mathematicians use logical thinking to reason and prove the process of inference.",
"prompt": "Which subject are you better at?"
},
{
"output": "Lu Xun's \"Diary of a Madman\" has problems in criticizing feudal ethics, mainly because its language style and content are too exaggerated and hyperbolic, or do not conform to social reality, causing problems in society's evaluation of human nature and social relations.\n\nLu Xun's novels usually use a humorous, satirical, and ironic technique, revealing some problems of feudal society in a very humorous way, while also criticizing the constraints and rules of feudal ethics.\n\nIn criticizing feudal ethics, Lu Xun's novels usually use a very exaggerated and hyperbolic technique, revealing some problems in social reality in a very humorous way. This technique makes the work seem very absurd and hollow, while also criticizing the social order and moral concepts of the time.\n\nLu Xun's novels usually use a very humorous, satirical, and ironic technique, revealing some problems of feudal society in a very humorous way. This technique makes the work seem very absurd and hollow, while also criticizing social relations of the time, making the work seem very absurd and hollow.\n\nTherefore, in criticizing feudal ethics, Lu Xun's novels often use a humorous, satirical, and ironic technique, revealing social problems in a very humorous way. This technique makes the work seem very absurd and hollow, while also criticizing the social order and moral concepts of the time.",
"prompt": "How does Lu Xun's \"Diary of a Madman\" criticize feudal ethics?"
},
{
"output": "First, it is necessary to determine the specific situation of the cough. Coughing is a manifestation of physical discomfort, mainly due to physical discomfort, including lung infections, pneumonia, etc. If the cough has persisted for two weeks, it is recommended to go to the hospital for an examination. The doctor may examine the lungs to check for inflammation or infection. If not, the doctor may recommend examinations such as a chest X-ray or blood tests. If necessary, the doctor can also perform blood tests, throat radiation examinations, etc. During the examination, care should be taken to avoid excessive force, not to drink alcohol or eat irritating foods, and to avoid contact with others. If the cough persists for two weeks, it is recommended to go to the hospital promptly for diagnosis and treatment of the condition.",
"prompt": "I have had a cough for two weeks. Do I need to go to the hospital for a check-up?"
},
{
"output": "The speed of light refers to the propagation speed of light in a vacuum, and its unit of measurement is the speed of light (c). In a vacuum, the speed of light is 299,792,458 meters per second, approximately 393,458 meters per second.\n\nThe speed of light is one of the fundamental constants in physics, indicating that the speed of light is always equal to 299,792,458 meters per second at any time. In relativity, the speed of light is defined as 9.87924/966.71 seconds, while in other scientific fields, such as astronomy and aerospace, the speed of light is defined as 393,458 meters per second.\n\nIn summary, the speed of light is a very important concept because it is a fundamental concept in physics and is widely used in science and technology.",
"prompt": "Explain the physical concept of the speed of light in detail."
},
{
"output": "Hangzhou has many specialty foods. Here are a few recommendations for you: 1. West Lake Vinegar Fish: This is a delicate, sweet and sour specialty snack, a signature dish of Hangzhou. 2. Dongpo Pork: This is one of Hangzhou's traditional famous dishes, with tender meat and a fragrant, sticky texture. 3. Dongpo Steamed Bun: This is a traditional Chinese pastry, crispy in texture with rich fillings. 4. Handmade Dumplings: These are dumplings mainly filled with meat, with soft fillings and a refreshing taste. 5. Dongpo Soup: This is a traditional soup dish, rich in broth and delicious in flavor. 6. Xiaolongbao (Soup Dumplings): These are small steamed buns with rich, fragrant fillings and a chewy, springy texture. 7. Handmade Baozi (Steamed Buns): This is a traditional West Lake snack, with rich and delicious fillings. 8. Stone Pot Rice: This is a traditional snack mainly based on rice, with rich and delicious fillings. The above foods are all very distinctive. You can choose to try them according to your taste.",
"prompt": "Recommend some Hangzhou specialty foods."
},
{
"output": "A Large Language Model (Generative Area) is a neural network model trained on large-scale datasets that can handle various tasks of natural language, such as machine translation, text classification, sentiment analysis, speech recognition, and text generation. The principle of this model is through continuous iterative learning, constantly updating and optimizing the model, enabling the model to gradually understand the input natural language, thereby generating more accurate and natural text. Specifically, Generative adopts a model based on Recurrent Neural Networks (RNN). Through pre-training, it compares the input natural language sequence with the pre-trained sequence data, thereby obtaining more accurate and natural text output. Generative performs excellently in language generation, capable of generating more fluent and natural text. At the same time, Generative can continuously improve its generation effect through iterative learning and optimization, thereby better meeting user needs.",
"prompt": "Please explain the concept of \"Large Language Model\" to me."
},
{
"output": "ChatGPT is a model based on artificial intelligence technology. It can be used to complete natural language processing tasks, such as translation, question answering, generation, and classification. The accuracy of ChatGPT can reach over 90%, which means it can quickly understand the questions raised by users and give correct answers. It can also handle various complexities in natural language, such as grammar and semantics. ChatGPT can also handle text generation and classification tasks, such as text summarization, question answering, and classification. In addition, it can handle speech and image processing tasks for text, such as speech recognition and text-to-speech, as well as sentiment analysis tasks, such as comment and public opinion analysis. Overall, ChatGPT is a powerful natural language processing tool that can effectively process natural language and handle text and speech data.",
"prompt": "How to understand ChatGPT?"
},
{
"output": "Introduce the history of theUnitedStates, please.",
"prompt": "Introduce the history of the United States, please."
}
],
"stage": "dpo",
"terminal_header": "### DPO Model\n\n\n(base) root@C.26849627:~/minimind$ python eval_model.py --load 0 --model 2\nMiniMind model parameters: 104.03M (million)\n[0] Auto test\n[1] Manual input\n0"
},
{
"arm": "qk_norm_muon",
"pair_count": 7,
"pairs": [
{
"output": "It studies the influence of the bourgeoisie on social structure, mode of production, culture, morality, etc., from a metaphysical perspective. He believes that the bourgeoisie is the main driver of social structure, mode of production, culture, morality, etc., and its behavior and decisions directly affect the formation and change of social structure.\n\nMarxist theory still holds a very important position in the contemporary era. 1. What are the basic principles of Marxism?\n2. What is the object of study of Marxist theory?\n3. What does Marxist theory influence?\n4. Does Marxist theory still play an important role in contemporary society?",
"prompt": "Basic principles of Marxism"
},
{
"output": "What are they? The main functions of the human brain are to process sensory information, including vision, hearing, movement, memory, emotion, language, thinking, and decision-making. The processing methods of the human brain vary from individual to individual, but most studies show that the human brain can achieve advanced cognitive functions, such as perception, learning, memory, thinking, emotion, and will.",
"prompt": "Main functions of the human brain"
},
{
"output": "How is it calculated? The principle of universal gravitation was discovered by the British physicist Newton in the 17th century. He discovered that the universal gravitation between objects is because they are all attracted to each other, and the magnitude of this attraction is related to the mass of the objects and the distance between them. This law explains why the gravitational force between objects is proportional to their masses and inversely proportional to the square of the distance between them. Newton's law of universal gravitation is also used to explain celestial motion and planetary orbits.",
"prompt": "The principle of universal gravitation is"
},
{
"output": "Mount Everest, located in the Himalayas, with an altitude of 8848 meters. Mount Everest, located in the Himalayas, is the highest mountain in the world. Its height is 8848 meters, and it has always been a desired destination for people on the border of Nepal and China. The height of Mount Everest has been attempted by many climbers, but only a few have been able to successfully summit. The height and difficulty of this mountain are extremely high, requiring great physical fitness, endurance, and skill to climb.",
"prompt": "The highest mountain in the world is"
},
{
"output": "Will its concentration decrease as the temperature rises? The concentration of carbon dioxide varies with different temperatures and air temperatures. However, in most cases, the concentration of carbon dioxide is related to its content in the atmosphere.\n\nAt higher temperatures, the concentration of carbon dioxide may decrease because carbon dioxide is a greenhouse gas, and its concentration can be negligible. For example, at high temperatures, the concentration of carbon dioxide may decrease, but at low temperatures, the concentration of carbon dioxide may increase.\n\nFurthermore, under different temperatures and humidity levels, the concentration of carbon dioxide may change. For example, at higher temperatures, the concentration of carbon dioxide may increase because the rise in temperature leads to an increase in water vapor content, thereby causing the concentration of carbon dioxide in the air to decrease.\n\nIn summary, the concentration of carbon dioxide is affected by many factors, and being affected by temperature in the atmosphere will cause the concentration of carbon dioxide to decrease.",
"prompt": "Carbon dioxide in the air"
},
{
"output": "How many? The largest animal on Earth is the blue whale (Balaenoptera musculus). Blue whales can grow up to 30 meters long and weigh up to 173 tons. They are currently the largest known animals and the largest animals on Earth.\nBlue whales are distributed in all major oceans around the world, in the waters surrounding Antarctica. They are the largest species of whale, with a body weight that can reach 200 tons. Blue whales are carnivorous, feeding on small planktonic animals.\nBesides the blue whale, the largest animal on Earth is the African elephant (Loxodonta afasciola), which is the largest land animal on Earth. African elephants can grow up to 4 meters tall and weigh over 6 tons.\nIn summary, the blue whale is the largest animal in the world, and they live in the oceans around the globe.",
"prompt": "The largest animal on Earth has"
},
{
"output": "What dishes are worth trying?\nWhat is the answer to this question? My favorite is Longjing Shrimp, and I must eat it every time I go to Hangzhou!",
"prompt": "Hangzhou's cuisine has"
}
],
"stage": "pretrain",
"terminal_header": "### Pretrain Model\n\n\n$ CUDA_VISIBLE_DEVICES=1 python eval_model.py --load 0 --model 0\nMiniMind model parameters: 104.03M (million)\n[0] Auto test\n[1] Manual input\n0"
},
{
"arm": "qk_norm_muon",
"pair_count": 9,
"pairs": [
{
"output": "I am an AI language model capable of text generation and answering various questions. I am trained for natural language processing, language understanding, natural language generation, dialogue systems, etc., and can provide users with efficient, accurate, and convenient services. My goal is to provide users with accurate, comprehensive, and professional knowledge, helping them solve various problems and improve their own performance and abilities.",
"prompt": "Please introduce yourself."
},
{
"output": "I am better at English, mathematics, science, and history. In these subjects, I can use English and English grammar, learn English grammar and spelling rules, and be familiar with various historical events and cultures. I am also very good at English, mathematics, science, and history.",
"prompt": "Which subject are you better at?"
},
{
"output": "Lu Xun's \"Diary of a Madman\" is a literary work that opposes feudal ethics. In this work, the protagonist is a mental patient who is given a fate of self-destruction and begins to destroy himself. However, the novel also depicts the cruelty and depravity of feudal ethics. In the novel, Lu Xun uses his critique of feudal ethics to reflect on this system. He believes that feudal ethics is not only a critique of feudal ethics but also a rebellion against society. In the novel, Lu Xun's \"Diary of a Madman\" criticizes the oppression and enslavement of feudal ethics and raises questions about feudal ethics. He believes that feudal ethics is a corruption of human nature and an exploitation of feudal ethics. In addition, the novel also criticizes the critique of modern culture by feudal ethics. Lu Xun believes that feudal ethics is a kind of \"banned book,\" prohibiting the publication and dissemination of literary works and requiring public plagiarism. This behavior seriously violates the legitimacy of feudal ethics and severely damages the image of feudal ethics. In summary, Lu Xun's \"Diary of a Madman\" criticizes the oppression and enslavement of feudal ethics and raises questions about feudal ethics. This critique reflects the corruption and moral decay within Chinese feudal society, as well as the critique of modern culture by feudal ethics.",
"prompt": "How does Lu Xun's \"Diary of a Madman\" criticize feudal ethics?"
},
{
"output": "If the cough has persisted for two weeks, the doctor may recommend further examinations. These examinations include hearing tests, lung CT scans, and chest X-rays to assess the condition of the respiratory system. If your cough lasts more than two weeks, the doctor may suggest you go to the hospital for a visit.",
"prompt": "I have had a cough for two weeks. Do I need to go to the hospital for a check-up?"
},
{
"output": "The speed of light is one of the most fundamental physical constants in physics. It represents the speed at which light travels in a vacuum, approximately 299,792,458 meters per second. The speed of light is the speed at which light propagates in a vacuum, with a value of 299,792,458 meters per second.\nThe physical concept of the speed of light includes the following aspects:\n1. Relativity of the speed of light: The speed of light is a constant that is not affected by the observer's state of motion.\n2. Equivalence of the speed of light: The speed of light is not affected by the observer's state of motion and is a constant, meaning it is not affected by the observer's position.\n3. Conservation of light energy: The speed of light is invariant in any inertial reference frame, meaning it is not affected by the observer's state of motion.\n4. Propagation speed of light: The speed of light in a vacuum is 299,792,458 meters per second, approximately 299,792,458 meters per second.\n5. Invariance of the speed of light: The speed of light is not affected by the observer's state of motion and is a constant, meaning the speed of light in a vacuum is not affected by the observer's position.\nThe importance of the speed of light in physics cannot be ignored. It is not only a fundamental physical constant of the speed of light but also one of the most important fundamental constants in physics.",
"prompt": "Please provide a detailed introduction to the physical concept of the speed of light."
},
{
"output": "Hangzhou's food culture is very rich. Here are a few famous specialty dishes: 1. Longjing Shrimp: This is a traditional Hangzhou dish made with fresh shrimp as the main ingredient, processed through steaming, frying, boiling, and other steps, resulting in a delicious taste. 2. Dongpo Pork: This is a traditional dish made with pork as the main ingredient, slowly braised with seasonings such as scallions, ginger, garlic, star anise, and cinnamon. 3. West Lake Vinegar Fish: This is a sweet and sour dish made with fish as the main ingredient, seasoned with vinegar, sugar, salt, and other seasonings. 4. Beggar's Chicken: This is a traditional dish made with chicken as the main ingredient, slowly braised with seasonings such as Sichuan peppercorns, ginger, and scallions. 5. Dongpo Eggs: This is a traditional dish made with eggs as the main ingredient, slowly braised with seasonings such as scallions, ginger, and salt.",
"prompt": "Recommend some Hangzhou specialty foods."
},
{
"output": "A \"Large Language Model\" refers to a language model capable of understanding and generating natural language text. It is a natural language processing technology based on neural networks. Building on the previously mentioned language models, it can perform tasks such as text generation, machine translation, and question-answering systems. Training a large language model requires a large amount of data and computational resources, but the quality and reliability of its training results can directly affect its performance in natural language processing tasks.",
"prompt": "Please explain the concept of \"Large Language Model\" to me."
},
{
"output": "ChatGPT is an artificial intelligence language model that can answer questions based on an understanding of human language. It can understand and generate natural language, but it is not in the form of a \"chat.\" It is a computer program that can communicate using natural language like a human.\nTo understand ChatGPT, you need to use natural language processing (NLP) techniques and deep learning algorithms. These algorithms are trained to understand inputs and outputs and can generate human-like text. This may involve extracting features from text, such as grammar, syntax, and contextual information, and then using this information to generate new text.\nTo use ChatGPT, you need to provide it with a large amount of text data, which can be structured data (such as text documents or web pages) or unstructured data (such as images or audio). Then, you can use deep learning algorithms, such as Recurrent Neural Networks (RNNs) or Transformers, to train the model to generate human-like text.\nOverall, ChatGPT is a powerful tool that can help you process natural language, but understanding its complexity and abstraction remains a challenging task.",
"prompt": "How to understand ChatGPT?"
},
{
"output": "- Introduce the history of the United States.- Please.",
"prompt": "Introduce the history of the United States, please."
}
],
"stage": "sft",
"terminal_header": "### SFT Model\n\n\n$ python eval_model.py --load 0 --model 1\nMiniMind model parameters: 104.03M (million)\n[0] Auto test\n[1] Manual input\n0"
},
{
"arm": "qk_norm_muon",
"pair_count": 9,
"pairs": [
{
"output": "I am an AI assistant developed based on natural language processing technology. I can answer various questions and provide various services, including but not limited to providing information, providing entertainment, and helping to solve various problems. I am constantly learning and evolving to better serve users.",
"prompt": "Please introduce yourself."
},
{
"output": "I am better at computer science and mathematics. In these subjects, I am particularly good at programming, including Java and Python. I can use these programming languages to write code, develop software applications, and solve real-world problems.",
"prompt": "Which subject are you better at?"
},
{
"output": "Lu Xun's \"Diary of a Madman\" is an article that criticizes the corruption of human nature by feudal ethics. In the article, Lu Xun criticizes the rebellion and oppression of feudal ethics. He believes that the criticism of human nature by feudal ethics is wrong. First, feudal ethics is a feudal system that controls people's behavior and thoughts, limiting people's freedom and dignity. The criticism of human nature by feudal ethics is an oppression and exploitation of feudal ethics, which has had a negative impact on the progress of China's modernization process. Second, the criticism of human nature by feudal ethics is an oppression and exploitation of feudal ethics. The criticism of human nature by feudal ethics is an oppression and exploitation of feudal ethics. The oppression of feudal ethics by feudal ethics is an oppression and exploitation of feudal ethics. Finally, the criticism of human nature by feudal ethics expressed in Lu Xun's \"Diary of a Madman\" is an oppression and exploitation of feudal ethics. The criticism of human nature by feudal ethics is an oppression and exploitation of feudal ethics. In short, Lu Xun's \"Diary of a Madman\" criticizes feudal ethics, with the oppression and exploitation of feudal ethics as the main direction of criticism, and proposes rebellion and oppression against feudal ethics. At the same time, Lu Xun also reflects on the oppression and exploitation of feudal ethics, expressing reverence and awe for feudal ethics.",
"prompt": "How does Lu Xun's \"Diary of a Madman\" criticize feudal ethics?"
},
{
"output": "If you have been coughing for two weeks, it is recommended to go to the hospital for a check-up. Usually, it is best to have a check-up under the guidance of a doctor. The doctor will evaluate your symptoms and medical history and may perform some basic physical examinations, such as auscultation, facial examination, and blood tests. If your symptoms persist and do not show significant improvement, you may need further examination at the hospital. In some cases, the doctor may recommend blood tests, lung function tests, chest X-rays, electrocardiograms, etc. If your cough symptoms do not show significant improvement, you may need to go to the hospital for further examination. Regardless of the severity of your symptoms, the doctor will develop a treatment plan based on your condition. If your symptoms are severe or last for a long time, it is recommended to seek medical attention as soon as possible.",
"prompt": "I have had a cough for two weeks. Do I need to go to the hospital for a check-up?"
},
{
"output": "The speed of light is a commonly used concept in physics, referring to the speed at which light propagates in a vacuum. Its value is 299,792,458 meters per second, which is the fastest speed in nature. The physical concept of the speed of light includes the following points: 1. The speed of light is a natural constant, i.e., 299792.9835478 seconds. 2. The speed of light is a constant, i.e., the speed at which light propagates in a vacuum, which is the fastest speed in nature. 3. The speed of light is an important concept, significant for fields such as physics, astronomy, and cosmology. 4. The unit of the speed of light is meters per second, representing the speed at which light propagates in a vacuum, approximately 299,792,458 meters per second. 5. The speed of light is an important physical constant, closely related to the wave nature of light and electromagnetic radiation. 6. The speed of light plays an important role in relativity, being one of the foundations of general relativity. The speed of light is also one of the foundations of special relativity, an important concept for describing the relativity of object motion and light. In summary, the speed of light is a commonly used concept in physics, significant for fields such as physics, astronomy, and cosmology.",
"prompt": "Please provide a detailed introduction to the physical concept of the speed of light."
},
{
"output": "Hangzhou has many famous specialty foods, such as West Lake Vinegar Fish, Longjing Shrimp, Beggar's Chicken, Dongpo Pork, etc. In addition, Hangzhou's Xiaolongbao (soup dumplings), Glutinous Rice Chicken, Longjing Shrimp Sweet and Sour Fish, and other delicacies are also not to be missed.",
"prompt": "Recommend some Hangzhou specialty foods."
},
{
"output": "A large language model, also known as a large language model, is a language model based on a statistical model used to generate natural language text. It learns the patterns of language elements such as vocabulary, grammar, and context to build a model that can generate text conforming to language rules. Large language models have a wide range of applications, such as machine translation, speech recognition, and natural language generation.",
"prompt": "Please explain the concept of \"Large Language Model\" to me."
},
{
"output": "ChatGPT is an artificial intelligence language model that uses machine learning algorithms to generate human-like text. ChatGPT learns language patterns and grammatical rules by training on large amounts of text data, enabling it to generate human-like responses. It is an advanced technology widely used in various applications, such as chatbots, intelligent assistants, and language translation. To understand ChatGPT, it is necessary to delve into its internal working principles, such as how it generates responses based on previous input and how it uses probability-based statistical methods to learn language patterns and grammatical rules.",
"prompt": "How to understand ChatGPT?"
},
{
"output": "Introduce the history of the United States, please.",
"prompt": "Introduce the history of the United States, please."
}
],
"stage": "dpo",
"terminal_header": "### DPO Model\n\n\n$ python eval_model.py --load 0 --model 2\nMiniMind model parameter count: 104.03M(illion)\n[0] Automatic test\n[1] Manual input\n0"
}
],
"experiment": "8-3",
"output_count": 49,
"schema_version": "exp8-3-retained-outputs-v1",
"source_report": "chapter8/MiniMind-pretrain/README.md",
"source_report_sha256": "a8e1df1a9ee5cf013995e9ff3b963621485a838c97456b9f30e65ea9fdf55d50",
"stages": [
"pretrain",
"sft",
"dpo"
]
}
chapter8/MiniMind-pretrain/validation/runs/exp8-3-training-report-20260731-v1/summary.json
+276
View File
@@ -0,0 +1,276 @@
{
"acceptance": {
"all_expected_outputs_retained": true,
"all_six_arm_stage_cells_retained": true,
"checkpoints_not_an_acceptance_artifact": true,
"eight_stage_balanced_blind_judgments": true,
"future_reproduction_commands_declared": true,
"historical_provenance_limitations_explicit": true,
"immutable_dataset_revision_lfs_hashes_and_sizes_frozen": true,
"immutable_source_revision_and_file_hashes_frozen": true,
"original_and_qk_norm_muon_compared": true,
"passed": true,
"pretrain_sft_and_dpo_compared": true,
"raw_historical_report_hashed": true,
"raw_judge_requests_responses_ids_usage_latency_retained": true,
"reported_loss_claims_qualified": true
},
"arm_averages": {
"original": {
"factuality": 1.375,
"instruction_following": 1.75,
"language_fluency": 3.0,
"overall": 2.0417
},
"qk_norm_muon": {
"factuality": 4.125,
"instruction_following": 3.0,
"language_fluency": 3.75,
"overall": 3.625
}
},
"experiment": "8-3",
"judge": {
"blind_seed": 730731,
"calls": 8,
"model": "doubao-seed-1-6-250615",
"provider": "ark",
"response_ids": [
"02178549583945856db6dee5d970b68ab3a378dc7e67e35390cf8",
"021785495839461d24b0b4d764165756d4018ab78dadfacc3782e",
"021785495839460982b29f8728970ed398f78ebc517c3ae19045a",
"021785495839459da6f4ee8e5443fabd9dcec9964b863512d1526",
"021785495872253a78e4bf9e905a6c19d35dac8ed03e6be35e22e",
"0217854958740424ebcb86fd234a9cf7780741b795c6db019ca22",
"0217854958861885bd2f0b9e6315ce45b86fb4e22ec749679fcc6",
"02178549588624031c26a7cf9ea611aa30972127acb8017b9f7cf"
],
"total_latency_ms": 285434.825,
"total_tokens": 15652
},
"limitations": [
"Historical checkpoints are intentionally not distributed and were not recreated in this audit.",
"The historical source revision, dataset byte identities, RNG state, and stepwise loss logs were not retained.",
"Frozen source/data revisions and the book lock define a future reproduction contract, not historical provenance.",
"The independent judge covers eight preregistered comparisons; all other retained outputs remain available for inspection.",
"The historical outputs are English translations in a bilingual report, so translation may affect the judge scores."
],
"per_case_arm_scores": {
"1": {
"original": {
"factual_errors": [
"In Nepali, its name means 'Goddess's Home' (incorrect; Nepali name Sagarmatha means 'Forehead of the Sky' or similar)"
],
"factuality": 3,
"instruction_following": 3,
"language_fluency": 3,
"rationale": "Correctly names Mount Everest, its location, and altitude, but contains a factual error about the Nepali name's meaning. Starts with an irrelevant 'which one?' (not following the prompt's statement structure) and has repetitive details (e.g., repeating altitude and location), leading to partial instruction following and flawed fluency."
},
"qk_norm_muon": {
"factual_errors": [],
"factuality": 5,
"instruction_following": 5,
"language_fluency": 3,
"rationale": "Accurately identifies Mount Everest as the highest mountain, with correct location (Himalayas) and altitude (8848 meters). No material factual errors. Directly answers the prompt, though with repetitive phrasing (e.g., repeating 'Mount Everest, located in the Himalayas') and a fragmentary opening sentence, reducing fluency."
}
},
"2": {
"original": {
"factual_errors": [
"States CO2 concentration is 'about 20% of air' (actual ~0.04%)",
"Refers to CO2 as a 'very important element' (it is a compound)",
"Claims CO2 is 'main gas for respiration in humans' (humans exhale CO2, do not use it for respiration)"
],
"factuality": 0,
"instruction_following": 2,
"language_fluency": 3,
"rationale": "Contains severe factual errors (e.g., 20% concentration), misclassifies CO2 as an element, and misrepresents its role in respiration. Attempts to correct a false claim but introduces major inaccuracies. Language is coherent but flawed."
},
"qk_norm_muon": {
"factual_errors": [
"Claims CO2 concentration 'can be negligible' at higher temperatures due to being a greenhouse gas (false; greenhouse properties don't reduce concentration)",
"Contradicts temperature effect (higher temp 'may decrease' then 'may increase because... thereby causing decrease')",
"Incorrectly concludes temperature causes CO2 concentration to decrease (no evidence for direct relationship)"
],
"factuality": 0,
"instruction_following": 2,
"language_fluency": 3,
"rationale": "Contains multiple contradictory and false claims about CO2 concentration and temperature; attempts to address the topic but with major errors. Language is understandable but has logical defects."
}
},
"3": {
"original": {
"factual_errors": [
"Claims matter exceeds speed of light",
"Incorrect formula $c^2=m^2$",
"Miscalculated speed (2997*2458 and 4.08 m/s)",
"States speed depends on light intensity",
"Falsely claims light propagation is 'limited to a tiny range'"
],
"factuality": 0,
"instruction_following": 1,
"language_fluency": 2,
"rationale": "Contains numerous material falsehoods about speed of light values, matter speed, and propagation; largely incoherent and fails to explain the concept."
},
"qk_norm_muon": {
"factual_errors": [],
"factuality": 5,
"instruction_following": 4,
"language_fluency": 4,
"rationale": "Accurately states the speed of light in a vacuum and key concepts like invariance; minimal redundancy but no material factual errors."
}
},
"4": {
"original": {
"factual_errors": [
"Claims ChatGPT is 'developed by Google' (it is developed by OpenAI)",
"States it 'uses speech recognition technology' (ChatGPT is primarily text-based, not focused on speech recognition)"
],
"factuality": 0,
"instruction_following": 3,
"language_fluency": 3,
"rationale": "Contains major factual errors about developer and core technology. Attempts to explain functionality but is incomplete (e.g., mid-sentence line break) and has redundant phrasing, making it partially understandable but flawed."
},
"qk_norm_muon": {
"factual_errors": [],
"factuality": 5,
"instruction_following": 5,
"language_fluency": 5,
"rationale": "Accurately identifies ChatGPT as an AI language model using NLP and deep learning (e.g., Transformers), explains training on text data. No material factual errors. Directly answers the task by explaining what it is and how it works. Fluent and coherent throughout."
}
},
"5": {
"original": {
"factual_errors": [
"mentions 'embers' which is irrelevant to and not part of US history"
],
"factuality": 0,
"instruction_following": 0,
"language_fluency": 3,
"rationale": "Output repeats the prompt with the irrelevant term 'embers', containing a material factual error, failing to introduce US history, and is understandable but nonsensical due to 'embers'."
},
"qk_norm_muon": {
"factual_errors": [],
"factuality": 5,
"instruction_following": 0,
"language_fluency": 3,
"rationale": "Output repeats the prompt without adding content, containing no material factual errors, failing to introduce US history, and is understandable but has awkward punctuation."
}
},
"6": {
"original": {
"factual_errors": [
"Incorrect approximate speed (393,458 m/s instead of ~299,792,458 m/s)",
"Nonsensical definition in relativity: '9.87924/966.71 seconds' (speed cannot be defined in seconds)",
"Incorrect unit statement: 'unit of measurement is the speed of light (c)' (unit should be meters per second)"
],
"factuality": 1,
"instruction_following": 2,
"language_fluency": 3,
"rationale": "Contains multiple severe factual errors: incorrect speed values, nonsensical definitions in relativity, and wrong unit description. Partially addresses the concept but is undermined by critical inaccuracies."
},
"qk_norm_muon": {
"factual_errors": [
"Incorrect unit in point 1: '299792.9835478 seconds' (seconds is a unit of time, not speed)"
],
"factuality": 3,
"instruction_following": 3,
"language_fluency": 2,
"rationale": "Has a notable unit error but correctly states the speed value. Repeats points excessively but provides more relevant details (relativity, scientific fields) than A."
}
},
"7": {
"original": {
"factual_errors": [
"Claims 'accuracy can reach over 90%' (no general 'accuracy' metric exists for ChatGPT; performance varies by task and lacks substantiation); incorrectly states it 'handles speech and image processing tasks' (ChatGPT is primarily text-based, with no native image processing capabilities)"
],
"factuality": 2,
"instruction_following": 3,
"language_fluency": 4,
"rationale": "Contains material factual errors (unsubstantiated accuracy claim, image processing misstatement); explains basic function/uses but not 'how it works'; text is coherent with minor repetition."
},
"qk_norm_muon": {
"factual_errors": [],
"factuality": 5,
"instruction_following": 5,
"language_fluency": 5,
"rationale": "No material factual errors; accurately explains ChatGPT as an AI language model using ML algorithms, training on text data to learn patterns, and working principles (response generation via input and probability-based methods); fully addresses 'what it is' and 'how it works' with coherent, natural language."
}
},
"8": {
"original": {
"factual_errors": [],
"factuality": 5,
"instruction_following": 0,
"language_fluency": 3,
"rationale": "Output repeats the prompt without introducing U.S. history (non-answer, instruction following 0). No factual content (no material errors, factuality 5). Language has a typo ('theUnitedStates' missing space), understandable with defects (fluency 3)."
},
"qk_norm_muon": {
"factual_errors": [],
"factuality": 5,
"instruction_following": 0,
"language_fluency": 5,
"rationale": "Output repeats the prompt without introducing U.S. history (non-answer, instruction following 0). No factual content (no material errors, factuality 5). Language is coherent and natural (fluency 5)."
}
}
},
"retained": {
"cells": 6,
"outputs": 49,
"selected_comparisons": 8
},
"schema_version": "exp8-3-summary-v1",
"scientific_findings": {
"blind_judge_overall_delta_qk_norm_muon_minus_original": 1.5833,
"blind_judge_prefers_qk_norm_muon_overall": true,
"reported_loss_comparison_retained_but_not_independently_recomputed": true,
"wins": {
"original": 0,
"qk_norm_muon": 7,
"tie": 1
}
},
"stage_averages": {
"original": {
"dpo": {
"factuality": 2.6667,
"instruction_following": 1.6667,
"language_fluency": 3.3333,
"overall": 2.5556
},
"pretrain": {
"factuality": 1.5,
"instruction_following": 2.5,
"language_fluency": 3.0,
"overall": 2.3333
},
"sft": {
"factuality": 0.0,
"instruction_following": 1.3333,
"language_fluency": 2.6667,
"overall": 1.3333
}
},
"qk_norm_muon": {
"dpo": {
"factuality": 4.3333,
"instruction_following": 2.6667,
"language_fluency": 4.0,
"overall": 3.6667
},
"pretrain": {
"factuality": 2.5,
"instruction_following": 3.5,
"language_fluency": 3.0,
"overall": 3.0
},
"sft": {
"factuality": 5.0,
"instruction_following": 3.0,
"language_fluency": 4.0,
"overall": 4.0
}
}
},
"status": "passed"
}
chapter8/MiniMind-pretrain/validation/runs/exp8-4-training-report-20260731-v1/manifest.json
+83
View File
@@ -0,0 +1,83 @@
{
"acceptance": {
"all_64_historical_outputs_retained": true,
"all_eight_configuration_cells_retained": true,
"checkpoints_not_an_acceptance_artifact": true,
"eight_image_aware_arm_blind_judgments": true,
"future_reproduction_commands_declared": true,
"historical_provenance_limitations_explicit": true,
"historical_report_content_hashed": true,
"immutable_dataset_clip_and_eval_image_inputs_frozen": true,
"immutable_original_and_improved_source_revisions_frozen": true,
"passed": true,
"raw_judge_requests_responses_ids_usage_latency_retained": true,
"request_images_match_pinned_sha256": true,
"same_eight_images_present_in_every_cell": true
},
"artifacts": [
{
"bytes": 22282,
"path": "retained_outputs.json",
"sha256": "6333571fcdb95a84bece9fdf65456f328e7e048342ede37f71e3e03c68c38df6"
},
{
"bytes": 10758,
"path": "reproduction_contract.json",
"sha256": "e530aa559f940a2034e79315432789bf017abf0fc247adba3fb5e802893ea1d7"
},
{
"bytes": 2596470,
"path": "judge_receipts.json",
"sha256": "c777a01a7a484d7a6184575be101c167c0007442144e2bcb1f7047cee0dbc5f0"
},
{
"bytes": 33310,
"path": "summary.json",
"sha256": "fa904dfee4e8dbdf0c9986db79b3eba18adc00b3bfd823e9a99fca10e69a3f6d"
},
{
"bytes": 2116,
"path": "report.md",
"sha256": "0906af01fcf1f6c11528215fe8767c6fd5cad0743286cb0cf94af0df300f3df6"
}
],
"checkpoint_policy": "not distributed; not an acceptance artifact",
"created_at": "2026-08-17T05:34:53.397311+00:00",
"experiment": "8-4",
"inputs": [
{
"bytes": 155255,
"path": "chapter8/MiniMind-pretrain/README.md",
"sha256": "a8e1df1a9ee5cf013995e9ff3b963621485a838c97456b9f30e65ea9fdf55d50"
},
{
"bytes": 14150,
"path": "pyproject.toml",
"sha256": "33cc27c2759f353663d6907f1a918a41a49fb8d3e4e28370d508136449c3b156"
},
{
"bytes": 1311181,
"path": "uv.lock",
"sha256": "347fc87f40526372c284c61e5374536ce8d2071936cef8844d350470ecdf0d0b"
},
{
"bytes": 42148,
"path": "chapter8/MiniMind-pretrain/validation/run_vlm_training_report_audit.py",
"sha256": "d59e8175ce19a1973005263f6f7e4642e1ee11402860b4fadf6a1fd192d4cc7d"
},
{
"bytes": 14797,
"path": "chapter8/MiniMind-pretrain/validation/validate_vlm_evidence.py",
"sha256": "4ce53d7000d7f45a8e8012db1989374fcf42f5f832fbe134aca098e826b95db5"
},
{
"bytes": 6800,
"path": "chapter8/MiniMind-pretrain/validation/test_vlm_training_report_audit.py",
"sha256": "6259638aae28d1035f9d597f8f5ea17c7a66bd6ba8e3759ed1c842c94c19d1e1"
}
],
"run_dir": "validation/runs/exp8-4-training-report-20260731-v1",
"run_id": "exp8-4-training-report-20260731-v1",
"schema_version": "exp8-4-manifest-v1",
"status": "passed"
}
chapter8/MiniMind-pretrain/validation/runs/exp8-4-training-report-20260731-v1/report.md
+26
View File
@@ -0,0 +1,26 @@
# Experiment 8-4 retained-training-report audit
## Result
Status: **passed**. The historical report retains 64 image descriptions across 8 configurations and the same 8 images. Each image was inspected by a real image-capable ARK judge together with all eight arm-blind captions.
| Configuration | Grounding | Hallucination control | Coverage | Specificity | Overall | Best count |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| without_muon_sft | 2.1250 | 1.3750 | 2.3750 | 1.7500 | 1.9062 | 1 |
| without_muon_pretrained | 1.8750 | 2.6250 | 1.5000 | 0.8750 | 1.7188 | 1 |
| muon_from_dpo_sft | 1.3750 | 1.1250 | 1.8750 | 1.7500 | 1.5312 | 2 |
| muon_from_sft_pretrained | 1.8750 | 2.0000 | 1.3750 | 0.8750 | 1.5312 | 2 |
| muon_from_sft_sft | 1.2500 | 1.1250 | 1.5000 | 1.2500 | 1.2812 | 1 |
| muon_from_pretrain_sft | 1.1250 | 0.5000 | 1.3750 | 1.1250 | 1.0312 | 0 |
| muon_from_dpo_pretrained | 1.0000 | 1.5000 | 0.8750 | 0.3750 | 0.9375 | 0 |
| muon_from_pretrain_pretrained | 0.7500 | 1.3750 | 0.7500 | 0.6250 | 0.8750 | 1 |
The highest descriptive judge mean was **without_muon_sft** at **1.9062**. Averaged across all four base configurations, full VLM SFT changed the score by **+0.1718** versus projection-only pretraining.
The isolated report comparison pairs original/SFT-base against QK-Norm+Muon/SFT-base at each VLM stage. QK-Norm and Muon still change together, so no Muon-only causal claim is made. All author-written qualitative claims remain historical observations rather than pass/fail gates.
## Provenance and reproduction boundary
`reproduction_contract.json` freezes separate pre-QK-Norm and QK-Norm+Muon MiniMind-V revisions, the corresponding base-LLM revisions, script-compatible VLM dataset Git-LFS objects, the CLIP weight object, all eight evaluation-image hashes, and future commands. These pins are not misrepresented as the historical checkout.
Training checkpoints remain local by book policy and are not acceptance artifacts. The accepted artifact is this content-hashed report, all 64 retained outputs, eight raw image-aware judge receipts, and explicit limitations.
chapter8/MiniMind-pretrain/validation/runs/exp8-4-training-report-20260731-v1/reproduction_contract.json
+178
View File
@@ -0,0 +1,178 @@
{
"checkpoint_policy": {
"acceptance_artifact": false,
"distributed_with_book": false,
"reason": "Training checkpoints are intentionally not distributed to readers.",
"required_artifact": "reproducible evidence-backed training report"
},
"experiment": "8-4",
"future_reproduction": {
"base_llm_source": {
"dependency": "Use the Experiment 8-3 data/commands to produce original-SFT and improved pretrain/SFT/DPO 768-dimension base checkpoints.",
"original_files_sha256": {
"eval_model.py": "b9f7ea9d7f517551362bbf2da8f1de006b8c734bcba774b2be752bc63cc4349d",
"model/model_minimind.py": "7cb069cb0cb0dfa123cf11ea394d0001270bc683c0a2dfe4120fc3b861ffc0a4",
"trainer/train_dpo.py": "5e556a3089e43681638cdbf5adafb9d085bb1de5e4ea8da3ee522dfae02e3599",
"trainer/train_full_sft.py": "a57422f1df80bf2867f31f3b4a646a92ac7f66729e98a3b32cd1ec4d6780cb8b",
"trainer/train_pretrain.py": "ddd122645a9f1043bc8dac69a81ac51d2df95df8745d25faed7963d38fedc328"
},
"original_revision": "6d160ea20b98324632c4447ee63ec7cfa9becd20",
"qk_norm_muon_files_sha256": {
"eval_model.py": "43930a4b55048a4a3ffa17eb78ae67d59582d639aa9365f0bbf41ba149128af8",
"model/model_minimind.py": "2d33988711c704be6a22c4c61489b23106a2340a7cb8b97ebe3e40f30819cbb0",
"trainer/train_dpo.py": "97f2c31cc8bc21a777e2efcb5e2fa35a49e4e9e3698db120148f8a0b2f678449",
"trainer/train_full_sft_muon.py": "acd0b7db5b1d8b25d3c3103f92d68a7d381f9322a1b33bbec34be3d005930bad",
"trainer/train_pretrain_muon.py": "fc83d07754ec3a8c156b6b8bfc0fd4326edecb72efabc5e08ae4ff5e3a7029bc"
},
"qk_norm_muon_revision": "8bdc5d97d5845a8c1ac2ed56a5b8b4c0d0fb0795",
"repository": "bojieli/minimind"
},
"commands": {
"dataset": "git clone https://huggingface.co/datasets/jingyaogong/minimind-v_dataset dataset-source && git -C dataset-source checkout --detach ac9d03a3fd26a2d8e74bda374d9a2ddba49e4c1b && cp dataset-source/{pretrain_data.jsonl,sft_data.jsonl} dataset/ && unzip dataset-source/pretrain_images.zip -d dataset && unzip dataset-source/sft_images.zip -d dataset",
"evaluation": "For every isolated out_dir, preserve both checkpoints, copy the selected *_muon_768.pth name to eval_vlm.py's pretrain_vlm_768.pth or sft_vlm_768.pth compatibility name when needed, then run python eval_vlm.py --load 0 --model_mode 0 and --model_mode 1 on the eight hash-pinned images with seed 1337.",
"improved_matrix": "For each BASE in pretrain,sft,dpo, install the corresponding Experiment-8-3 QK-Norm+Muon 768-dimension checkpoint as runs/muon-from-$BASE/out/llm_768.pth, then run train_pretrain_vlm_muon.py and train_sft_vlm_muon.py with the same four-epoch data arguments in that isolated out_dir.",
"improved_source": "git clone https://github.com/bojieli/minimind-v.git sources/qk-norm-muon-minimind-v && git -C sources/qk-norm-muon-minimind-v checkout --detach ead791c530fa5f9a3549dbfe9e11ec732d18d2e5",
"original_pretrain_vlm": "install -m 0644 <exp8-3-original-sft-768.pth> runs/original/out/llm_768.pth && cd trainer && torchrun --nproc_per_node=8 train_pretrain_vlm.py --out_dir ../runs/original/out --epochs 4 --hidden_size 768 --num_hidden_layers 16 --data_path ../dataset/pretrain_data.jsonl --images_path ../dataset/pretrain_images --use_wandb",
"original_sft_vlm": "cd trainer && torchrun --nproc_per_node=8 train_sft_vlm.py --out_dir ../runs/original/out --epochs 4 --hidden_size 768 --num_hidden_layers 16 --data_path ../dataset/sft_data.jsonl --images_path ../dataset/sft_images --use_wandb",
"original_source": "git clone https://github.com/bojieli/minimind-v.git sources/original-minimind-v && git -C sources/original-minimind-v checkout --detach 765908051d0837d60cecfb93f8390334e2e55f1e",
"vision_encoder": "git clone https://huggingface.co/openai/clip-vit-base-patch16 model/vision_model/clip-vit-base-patch16 && git -C model/vision_model/clip-vit-base-patch16 checkout --detach 57c216476eefef5ab752ec549e440a49ae4ae5f3"
},
"environment": {
"book_lock": "uv.lock",
"book_pyproject": "pyproject.toml",
"boundary": "The book lock freezes a future Python environment; CUDA, drivers, and the historical GPU image were not retained.",
"install": "uv sync --locked --python 3.12 --extra ch7 --extra dev"
},
"evaluation_images": {
"Astronaut-Space.jpg": {
"sha256": "f466cdafecbdb85d2bad586896db5db3313afe18f9b3505667756cd25b747747",
"source_filename": "太空宇航员-Astronaut-Space.jpg"
},
"Bicycle-Flowers.jpg": {
"sha256": "44fae0fafcd52c20b9bcaded897facbff00f61019cdd0aea543addf8499ad899",
"source_filename": "自行车鲜花-Bicycle-Flowers.jpg"
},
"Chair-Elderly-Reading.jpg": {
"sha256": "8fe91a90e837c33230d21cfe7ba5020e71b3ae99ac4c3fbd6d32cb54f51def53",
"source_filename": "椅子老人看书-Chair-Elderly-Reading.jpg"
},
"Dog-Woman-Sea.jpg": {
"sha256": "ba90d8b8738a44eac70811be5c89f767492b167ad4f6f6c31aa4591837d7e3dc",
"source_filename": "小狗美女海边-Dog-Woman-Sea.jpg"
},
"Panda-Grassland.jpg": {
"sha256": "0b7610a881039f0effdbfa46e9bb189132443d3ce2956856e8adf66d1ca22f8c",
"source_filename": "熊猫草地-Panda-Grassland.jpg"
},
"Rainbow-Falls.jpg": {
"sha256": "1c8b74debaceb2e0bb6171b182084afe49288a0cc8089eb91eac69d067c27b10",
"source_filename": "彩虹瀑布-Rainbow-Falls.jpg"
},
"city-traffic.jpg": {
"sha256": "73e90d82fbc5b1cf43b40de782b443f93f43a34e66b8ddebf3146d5dc1f83e00",
"source_filename": "城市车水马龙-city-traffic.jpg"
},
"dance.jpg": {
"sha256": "939e3132c8d3aec81f66f8aa928b476aaa25e00d94f1097f4974e73c913d5d8c",
"source_filename": "舞蹈-dance.jpg"
}
},
"vision_encoder": {
"file": {
"bytes": 598641023,
"lfs_sha256": "ec89c7b09c749a60aae3c9cd910516f24b58214a7df060b48962d14c469cfbf0",
"path": "pytorch_model.bin"
},
"repository": "openai/clip-vit-base-patch16",
"revision": "57c216476eefef5ab752ec549e440a49ae4ae5f3"
},
"vlm_dataset": {
"files": {
"pretrain_data.jsonl": {
"bytes": 134315765,
"lfs_sha256": "abc9f2ba44190646692fbe7e2b49c366c5045490989fb32d2c5e960dd0ee10e4"
},
"pretrain_images.zip": {
"bytes": 2614907051,
"lfs_sha256": "64d56cee145bed75bc7f94c9cbf58882c41c4a0fea993014e27de7490b49e8b7"
},
"sft_data.jsonl": {
"bytes": 173137988,
"lfs_sha256": "c1993d38c3a22a8bdfee65affc82d6559e5bb62e785b0f21c9151c75116151fc"
},
"sft_images.zip": {
"bytes": 1026332147,
"lfs_sha256": "89ee34facc6793c51613613e0b10cac078942282f5fdec48d85751c6224bc3c2"
}
},
"repository": "jingyaogong/minimind-v_dataset",
"revision": "ac9d03a3fd26a2d8e74bda374d9a2ddba49e4c1b",
"selected_for_jsonl_script_compatibility": true
},
"vlm_source": {
"not_claimed_as_historical_revisions": true,
"original_files_sha256": {
"dataset/lm_dataset.py": "df20d57460d2845841ddf2e0faced1af1f7ec169e7fda3cd50fe3b2854288a92",
"eval_vlm.py": "9d883e4adbab0a7b88fd0cb9034132559a365387ec273ac4811cdd5ad28d5cda",
"model/model_minimind.py": "105429e93dcbe87145264d72d46a6add7639666036e999628c76ae50582507dc",
"model/model_vlm.py": "4ee42b298db68f30fbfa06d7686aa375d41a697628c770d0a134bca40ca9ea80",
"model/tokenizer.json": "d98595c6aef70d95f72748582fb9b4f53d76dd58c1ae1dd702ad7c84e1caf5e4",
"model/tokenizer_config.json": "dbbdb7eea33aba5c2608471494c93f650a2cf46fbe4a7489e531537ddadee746",
"requirements.txt": "a9bddf49d3ccbc9f8a2508ea039aebc0b996dccb0d3618d1b119af53a5d49869",
"trainer/train_pretrain_vlm.py": "4d30d54a940ae2eced204971cc03aafb3eb41a5c84c9f033cfdf162e63924a4d",
"trainer/train_sft_vlm.py": "8e3b920a6a135eb126bbeea07e2db748729cdd86050925b282a80537bc324e5f"
},
"original_revision": "765908051d0837d60cecfb93f8390334e2e55f1e",
"qk_norm_muon_files_sha256": {
"dataset/lm_dataset.py": "df20d57460d2845841ddf2e0faced1af1f7ec169e7fda3cd50fe3b2854288a92",
"eval_vlm.py": "9d883e4adbab0a7b88fd0cb9034132559a365387ec273ac4811cdd5ad28d5cda",
"model/model_minimind.py": "4771bc4b2ac367a6e6415c42c30bcdb54bec0397708f87de3c390042680b1e9e",
"model/model_vlm.py": "4ee42b298db68f30fbfa06d7686aa375d41a697628c770d0a134bca40ca9ea80",
"model/tokenizer.json": "e489029175fb3f94b8211a120a72a2ee41a664db65b828d077c7bde989c845a9",
"model/tokenizer_config.json": "190cc4738bac3b6f6b563376019c581b320fdb0260a03b9d5ab806296c8c6bb8",
"requirements.txt": "a9bddf49d3ccbc9f8a2508ea039aebc0b996dccb0d3618d1b119af53a5d49869",
"trainer/muon.py": "00c2c6a225edeb55433df0724c3c74f6ff98ac4b2cc73c4aafcff686824f6267",
"trainer/train_pretrain_vlm_muon.py": "f39af354c588747d9d5e522c9374a7f59a35d57aa649da74957da67d95d25bc6",
"trainer/train_sft_vlm_muon.py": "1fd56b3e8bed2714b4d10ceba5d57ada0b95d00dfbfb514481498fef0c0dd03d"
},
"qk_norm_muon_revision": "ead791c530fa5f9a3549dbfe9e11ec732d18d2e5",
"repository": "bojieli/minimind-v"
}
},
"historical_evidence_boundary": {
"claim": "The author's report establishes that eight VLM configurations were evaluated on eight images. It does not establish byte identity of the historical code, datasets, base/VLM checkpoints, or RNG state.",
"eight_historical_cells_and_64_outputs_retained": true,
"historical_base_checkpoint_hashes_retained": false,
"historical_dataset_hashes_retained": false,
"historical_rng_and_stepwise_logs_retained": false,
"historical_source_revisions_retained": false,
"historical_training_executed": true,
"historical_vlm_checkpoint_hashes_retained": false
},
"reported_training_design": {
"base_llm_stages": [
"pretrain",
"sft",
"dpo"
],
"parameter_count_millions": {
"original": 104.622,
"qk_norm_muon": 104.625
},
"projection_pretraining_freezes_llm": true,
"reported_epochs": 4,
"seed_in_current_source": 1337,
"sft_unfreezes_full_model": true,
"source_verified_mechanisms": {
"improved_revision_has_qk_norm_before_rope": true,
"improved_revision_uses_muon_for_selected_2d_weights": true,
"original_revision_precedes_qk_norm_commit": true,
"vision_encoder_is_frozen_clip": true
},
"vlm_stages": [
"pretrained",
"sft"
]
},
"schema_version": "exp8-4-reproduction-contract-v1"
}
chapter8/MiniMind-pretrain/validation/runs/exp8-4-training-report-20260731-v1/retained_outputs.json
+366
View File
@@ -0,0 +1,366 @@
{
"cell_count": 8,
"cells": [
{
"architecture": "original",
"base_llm_stage": "sft",
"config": "without_muon_pretrained",
"historical_command": "$ python eval_vlm.py --load 0 --model 0",
"output_count": 8,
"outputs": [
{
"image": "Rainbow-Falls.jpg",
"output": "Water droplets on the rainbow bridge"
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "A young person decorating the beach with a dog's head."
},
{
"image": "dance.jpg",
"output": "This dancer is performing on stage."
},
{
"image": "Astronaut-Space.jpg",
"output": "An astronaut is performing a spacewalk in space."
},
{
"image": "city-traffic.jpg",
"output": "On the city streets at nightfall."
},
{
"image": "Panda-Grassland.jpg",
"output": "In the zoo, a panda is eating bamboo."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "Using old bicycle tires and wheels on an old bicycle, adding flowers and decorations to the bicycle."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "An elderly woman reading a book in the park."
}
],
"vlm_stage": "pretrained"
},
{
"architecture": "original",
"base_llm_stage": "sft",
"config": "without_muon_sft",
"historical_command": "$ python eval_vlm.py --load 0 --model 1",
"output_count": 8,
"outputs": [
{
"image": "Rainbow-Falls.jpg",
"output": "This picture depicts a huge, colorful giant waterfall located next to a highway. From the image, you can see a massive and majestic giant waterfall. A giant rainbow flag hangs above the waterfall, adding visual appeal. The waterfall is surrounded by beautiful scenery, adding beauty to this picturesque landscape."
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "This image depicts a woman sitting on a beach with her dog beside her, while her dog sits on a blue and white checkered blanket."
},
{
"image": "dance.jpg",
"output": "The picture depicts a woman performing a dance on stage, her dance steps soaring high above the stage. Several chairs are placed on the stage, one on the left side of the frame, another on the right, and one more on the right side.\nMultiple chairs are placed on the stage, one near the left, another on the right. There is also a clock on the stage, possibly to help the performer keep time."
},
{
"image": "Astronaut-Space.jpg",
"output": "The image shows a huge spaceship that looks like a giant commercial airplane. This airplane is in the center of the frame, flying over a small hill and a barn. The airplane looks like a hangar, filled with all sorts of things.\nAround the airplane, some people might be observing this large commercial aircraft. Some of them seem to be watching the pilot's actions, while others might be admiring the details of the aircraft or enjoying the scene."
},
{
"image": "city-traffic.jpg",
"output": "This picture depicts a busy city street with heavy traffic, several cars and a truck moving. Tall buildings and streetlights illuminate the area, creating a vibrant and lively atmosphere.\nOn the street, various vehicles can be seen, including several cars and a truck. Some pedestrians also appear, possibly going about their daily activities or navigating the urban environment.\nOverall, this image captures a vibrant urban environment where vehicles, pedestrians, and traffic lights together create a lively atmosphere."
},
{
"image": "Panda-Grassland.jpg",
"output": "The image depicts a black and white panda wearing glasses, a long-haired black and white panda sitting on the grass. This panda seems to be staring at the camera, it looks like it is looking at the camera's image. This panda seems to be lying on the grass resting, possibly to avoid the sun or rain."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "The image shows a bicycle parked outside a building, with a row of vases and bouquets visible in the background. The bicycle occupies a considerable portion of the frame, extending from left to right. Throughout the scene, there are three potted plants, one near the left, another on the right, and a third on the left.\nBesides the bicycle, there are two people in the scene, one standing on the right and the other near the left side of the image."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "The image depicts an elderly man wearing glasses sitting on a park bench. He seems to be reading a book, possibly enjoying the outdoor environment. Several cars are parked nearby, one in the center of the image and another on the right. There is also a bench in the background, right next to the person sitting on the bench."
}
],
"vlm_stage": "sft"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "dpo",
"config": "muon_from_dpo_pretrained",
"historical_command": "$ python eval_vlm.py --load 0 --model 0",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "Night streets with elevators and sidewalks."
},
{
"image": "Astronaut-Space.jpg",
"output": "Spaceman sent to a new Earth"
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "You have a small ball in your palm, you throw it into the water."
},
{
"image": "Rainbow-Falls.jpg",
"output": "Viewed on the water, water landscape"
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "An elderly person sitting on a park bench."
},
{
"image": "Panda-Grassland.jpg",
"output": "This photo captures the cuteness and loveliness of this little white bear."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "Riding a bicycle on the street, bicycles everywhere, I saw many cars, this is my bicycle"
},
{
"image": "dance.jpg",
"output": "Performed by actors performing on stage"
}
],
"vlm_stage": "pretrained"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "dpo",
"config": "muon_from_dpo_sft",
"historical_command": "$ python eval_vlm.py --load 0 --model 1",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "This picture depicts a busy city street filled with skyscrapers and vehicles. At least twelve cars are driving on the street, some parked on the side, while others are moving.\nSeveral traffic lights can be seen, one on the left side of the street, another in the middle, and one on the right. Additionally, a bus is heading towards the center of the frame, adding to the lively atmosphere of this bustling city."
},
{
"image": "Astronaut-Space.jpg",
"output": "The person in the picture is wearing a black spacesuit and glasses, sitting on what looks like a space shuttle. They seem to be looking at a space shuttle flying in front of them, which is located on the right side of the scene.\nBesides the space shuttle, there are two other small boats in the picture, one on the left and another on the right. A smaller plane can also be seen in the background, and the person is located on the far right."
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "The image depicts a woman sitting on a bench by the beach, holding a small dog outside to sunbathe. She is wearing a dress and seems to be enjoying the outdoors.\nIn the background, several other people can be seen scattered on the beach, some closer to the water and others farther away. They all look happy, possibly spending a good time with this woman."
},
{
"image": "Rainbow-Falls.jpg",
"output": "The image shows a beautiful fountain surrounded by mist above it. The fountain is located above a high mountain, adding a magnificent background to the scene. An umbrella can be seen on the water, providing shade for visitors.\nThe terrain around the fountain consists of hills and mountains, offering a picturesque landscape for visitors. The presence of the fountain, water features, and surrounding mountains creates a serene and awe-inspiring atmosphere, making the area an ideal resort destination for photography enthusiasts and outdoor lovers."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "The image shows an elderly man sitting on a park bench, sitting in front of a book, holding it in his hands. The person looks relaxed and casual, possibly enjoying the outdoors.\nThere are several trees in the park, adding a natural atmosphere to the scene. Multiple benches are placed in different locations in the park for people to sit and enjoy the surroundings."
},
{
"image": "Panda-Grassland.jpg",
"output": "The protagonist of the picture is a giant panda with black and white markings, standing on a lush, green grassland. This giant panda seems to be enjoying the warmth of the sun and the grass."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "The image shows a green bicycle parked on a lane with yellow flowers, loaded with colorful blossoms. The bicycle is parked in front of a building, with two adjacent buildings forming a stark contrast.\nBesides the bicycle and flowers, there is a cup placed on the ground, possibly on or near the sidewalk."
},
{
"image": "dance.jpg",
"output": "In the image, a woman in a black dress is standing on a stage. She is performing on stage, surrounded by several people watching her, some of whom are holding mobile phones. These phones are scattered throughout the scene, possibly taken to capture the performance or communicate with the audience."
}
],
"vlm_stage": "sft"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "pretrain",
"config": "muon_from_pretrain_pretrained",
"historical_command": "$ python eval_vlm.py --load 0 --model 0",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "A bus traveling from the city center to a tall building"
},
{
"image": "Astronaut-Space.jpg",
"output": "🤖️: A little boy on a spaceship"
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "A dog walking by the seaside"
},
{
"image": "Rainbow-Falls.jpg",
"output": "Water droplets falling from a cave"
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "A young man reading a book on an elderly person's bed."
},
{
"image": "Panda-Grassland.jpg",
"output": "A bear with a layer of cute black-and-white fur"
},
{
"image": "Bicycle-Flowers.jpg",
"output": "In a park in Paris, a little boy rides a bicycle along the riverbank."
},
{
"image": "dance.jpg",
"output": "Performers wearing colorful costumes and fashionable outfits during a performance."
}
],
"vlm_stage": "pretrained"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "pretrain",
"config": "muon_from_pretrain_sft",
"historical_command": "$ python eval_vlm.py --load 0 --model 1",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "This image depicts a bustling city street at night, with high-rise buildings and heavy traffic. Cars are driving in different directions, creating a vibrant urban scene. Cars, trucks, and motorcycles flow continuously, showing the bustling activity on this street.\nA tall building stands prominently in the frame, forming part of the city skyline. The unique architectural style and scale of this building catch the attention of passersby, who might be drawn to appreciate its aesthetic value."
},
{
"image": "Astronaut-Space.jpg",
"output": "The image features a large spaceship or spacecraft, equipped with electronic devices and flight instruments. It appears to be from a museum or exhibition space, as various objects are displayed inside. These objects vary in size and seem to have been created by different professionals or scientists for visitors to explore."
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "In the picture, a woman sits on a beach bench holding a brown-and-white puppy. The puppy looks very happy, enjoying its time on the sand. In the background, two other people can be seen, also sitting on the same bench.\nBesides the woman and the puppy, various items appear in the scene, such as two handbags, a bench near the dog, a chair on the left, and a chair near the center of the scene."
},
{
"image": "Rainbow-Falls.jpg",
"output": "This image shows a stunning scene of rough seas extending to the horizon under a cloudy sky. A large mountain with a white dome stands in the center of the frame, becoming the focal point. The sea seems to be crashing against the horizon, creating an impression of strong waves.\nIn this scene, several people are scattered around, possibly enjoying the sea and the spectacular sky. Some of these people are closer to the coast, while others are farther away, likely admiring the breathtaking view."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "An elderly man wearing glasses sits at a table in a library, surrounded by cluttered bookshelves and various books. He is sitting in a chair, focused on a book, possibly meditating or deep in thought. Behind him, a potted plant can be seen, adding a touch of greenery to the scene.\nThe library environment is very crowded, with bookshelves filled with all kinds of books, covering almost the entire wall. These books vary in size and position, some even overturned, giving the library a busy atmosphere."
},
{
"image": "Panda-Grassland.jpg",
"output": "In the picture, a large black-and-white bear sits on green grass, with a few flowers scattered around. It seems to be enjoying time outdoors, possibly in a grove or on the grassland."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "The picture shows an outdoor garden with a bicycle parked between two other bicycles. Each bicycle has a basket filled with bright flowers. These flowers are arranged in various vases, adding color and vitality to the overall scene.\nIn the background, several people are in the garden, possibly enjoying a nice day. Some of them are near the bicycles, while others are elsewhere in the garden. Besides the bicycles, two cars are parked nearby, one near the foreground and the other farther away."
},
{
"image": "dance.jpg",
"output": "This image depicts a woman in elegant clothing walking down a wide walkway in the dark. She seems to be attending a dance, as she holds an umbrella, possibly to shield from rain or sunlight.\nChairs are placed along the walkway, likely for the audience to sit and enjoy the dance. Several kites can be seen in the picture, adding a joyful atmosphere to the dance."
}
],
"vlm_stage": "sft"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "sft",
"config": "muon_from_sft_pretrained",
"historical_command": "$ python eval_vlm.py --load 0 --model 0",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "Monitoring of nighttime traffic lights"
},
{
"image": "Astronaut-Space.jpg",
"output": "A series of conversations between \"our ship\" and \"our ship.\""
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "A cute woman in a white long dress and a cute pet dog"
},
{
"image": "Rainbow-Falls.jpg",
"output": "From the mountaintop, water glistens in the sunlight."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "An elderly person sitting on a bench in the park"
},
{
"image": "Panda-Grassland.jpg",
"output": "A cute zebra in the woods"
},
{
"image": "Bicycle-Flowers.jpg",
"output": "This video uses a horse working in a car."
},
{
"image": "dance.jpg",
"output": "A dancer performing on stage, wearing a stylish outfit."
}
],
"vlm_stage": "pretrained"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "sft",
"config": "muon_from_sft_sft",
"historical_command": "$ python eval_vlm.py --load 0 --model 1",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "This photo captures a busy city street at night, with cars and trucks on the road. The street is lined with high-rise buildings, creating a bustling atmosphere. The street is crowded with traffic, including multiple cars and a truck. Some of these vehicles are stationary, while others are moving.\nA large number of high-rise buildings appear in the frame, highlighting the urban environment. These buildings vary in size and shape, forming the architectural landscape of the area. This nighttime photo emphasizes the dynamic nature of city life, with various elements combining to create the character of a night-time metropolis."
},
{
"image": "Astronaut-Space.jpg",
"output": "The scene in the image shows a soldier sitting on a blue ship, which contains a large TV screen. The soldier appears to be looking at a smartphone screen, possibly for monitoring or communicating with the outside world.\nThere are several other people on the ship, possibly crew members or scientists. Some of them are scattered around the ship, while others are near the TV screen. Additionally, two TV remote controls are placed on the ship, indicating they might be monitoring or interacting with the equipment."
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "In the photo, a woman sits on the beach holding a large brown dog. She seems to be posing with the dog for the picture. The dog is on the left side of the frame.\nIn the background, multiple figures can be seen scattered across the beach. Some of them appear to be enjoying the beach, while others might be sightseeing or socializing."
},
{
"image": "Rainbow-Falls.jpg",
"output": "This image showcases an impressive fountain landscape, towering high on a grassy hill. The fountain occupies a significant portion of the frame, extending from the center to the far right, looking like a massive work of art.\nThe surroundings of the fountain, including the hill below and some trees in the background, provide a picturesque backdrop for this impressive fountain."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "In the picture, an elderly man sits on a park bench reading a newspaper. He seems to be enjoying a peaceful time in the park, perhaps resting in the afternoon or pondering something.\nIn the background, multiple cars can be seen parked near the park, creating a tranquil atmosphere. Additionally, a person appears at the top of the frame, possibly watching the man read the newspaper, perhaps enjoying nature or waiting for someone."
},
{
"image": "Panda-Grassland.jpg",
"output": "The picture shows a small black-and-white panda standing on a patch of grass. This cute panda looks very relaxed and comfortable."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "The picture shows a green bicycle parked next to a bench covered with a green tablecloth. This bench seems designed for gatherings, as it is near a potted plant.\nBesides the bicycle and the potted plant, a car is parked on the right side of the scene. A stop sign can be seen in the background, adding an urban touch to the image."
},
{
"image": "dance.jpg",
"output": "The image shows a woman in a tuxedo standing on a stage, holding a microphone, seemingly delivering an engaging performance. She looks like an actor or performer, with several people around her, possibly the audience or participants.\nVarious items are placed on the stage, including a wine glass and several bottles. Some of these bottles are near the stage, while others are scattered in the background. The scene captures an event on stage, with the actor or performer delivering a memorable show for the audience."
}
],
"vlm_stage": "sft"
}
],
"configs": [
"without_muon_pretrained",
"without_muon_sft",
"muon_from_dpo_pretrained",
"muon_from_dpo_sft",
"muon_from_pretrain_pretrained",
"muon_from_pretrain_sft",
"muon_from_sft_pretrained",
"muon_from_sft_sft"
],
"experiment": "8-4",
"images": [
"Rainbow-Falls.jpg",
"Dog-Woman-Sea.jpg",
"dance.jpg",
"Astronaut-Space.jpg",
"city-traffic.jpg",
"Panda-Grassland.jpg",
"Bicycle-Flowers.jpg",
"Chair-Elderly-Reading.jpg"
],
"output_count": 64,
"schema_version": "exp8-4-retained-outputs-v1",
"source_report": "chapter8/MiniMind-pretrain/README.md",
"source_report_sha256": "a8e1df1a9ee5cf013995e9ff3b963621485a838c97456b9f30e65ea9fdf55d50"
}
chapter8/MiniMind-pretrain/validation/runs/exp8-4-training-report-20260731-v1/summary.json
+858
View File
@@ -0,0 +1,858 @@
{
"acceptance": {
"all_64_historical_outputs_retained": true,
"all_eight_configuration_cells_retained": true,
"checkpoints_not_an_acceptance_artifact": true,
"eight_image_aware_arm_blind_judgments": true,
"future_reproduction_commands_declared": true,
"historical_provenance_limitations_explicit": true,
"historical_report_content_hashed": true,
"immutable_dataset_clip_and_eval_image_inputs_frozen": true,
"immutable_original_and_improved_source_revisions_frozen": true,
"passed": true,
"raw_judge_requests_responses_ids_usage_latency_retained": true,
"request_images_match_pinned_sha256": true,
"same_eight_images_present_in_every_cell": true
},
"best_counts": {
"muon_from_dpo_pretrained": 0,
"muon_from_dpo_sft": 2,
"muon_from_pretrain_pretrained": 1,
"muon_from_pretrain_sft": 0,
"muon_from_sft_pretrained": 2,
"muon_from_sft_sft": 1,
"without_muon_pretrained": 1,
"without_muon_sft": 1
},
"config_averages": {
"muon_from_dpo_pretrained": {
"coverage": 0.875,
"grounding_accuracy": 1.0,
"hallucination_control": 1.5,
"overall": 0.9375,
"visual_specificity": 0.375
},
"muon_from_dpo_sft": {
"coverage": 1.875,
"grounding_accuracy": 1.375,
"hallucination_control": 1.125,
"overall": 1.5312,
"visual_specificity": 1.75
},
"muon_from_pretrain_pretrained": {
"coverage": 0.75,
"grounding_accuracy": 0.75,
"hallucination_control": 1.375,
"overall": 0.875,
"visual_specificity": 0.625
},
"muon_from_pretrain_sft": {
"coverage": 1.375,
"grounding_accuracy": 1.125,
"hallucination_control": 0.5,
"overall": 1.0312,
"visual_specificity": 1.125
},
"muon_from_sft_pretrained": {
"coverage": 1.375,
"grounding_accuracy": 1.875,
"hallucination_control": 2.0,
"overall": 1.5312,
"visual_specificity": 0.875
},
"muon_from_sft_sft": {
"coverage": 1.5,
"grounding_accuracy": 1.25,
"hallucination_control": 1.125,
"overall": 1.2812,
"visual_specificity": 1.25
},
"without_muon_pretrained": {
"coverage": 1.5,
"grounding_accuracy": 1.875,
"hallucination_control": 2.625,
"overall": 1.7188,
"visual_specificity": 0.875
},
"without_muon_sft": {
"coverage": 2.375,
"grounding_accuracy": 2.125,
"hallucination_control": 1.375,
"overall": 1.9062,
"visual_specificity": 1.75
}
},
"experiment": "8-4",
"isolated_original_vs_qk_norm_muon_from_sft": {
"pretrained": {
"delta": -0.1876,
"original": 1.7188,
"qk_norm_muon_from_sft": 1.5312
},
"sft": {
"delta": -0.625,
"original": 1.9062,
"qk_norm_muon_from_sft": 1.2812
}
},
"judge": {
"blind_seed": 740731,
"calls": 8,
"image_aware": true,
"model": "doubao-seed-1-6-250615",
"provider": "ark",
"response_ids": [
"021785497883895355cac89e6d983ae8d30678a5b54f9afc7a30b",
"0217854978839024aa8fafbe9055982037cfb2f5ec29c229036d1",
"021785497883901cbe31712b8a2594f4e458beea699b0faabecc8",
"0217854978838946c88a39b89e3e3930a4a1be3bade8084753bc5",
"0217854979402514aa8fafbe9055982037cfb2f5ec29c22876438",
"021785497944549bdf0d8876ec55e469c432250c926544c9a9a49",
"0217854979562513b2c90331db17702ff1ef9ef62383e60dd8c12",
"02178549797786577f9606cef83fa80b3d313b2e208ca1cdfc167"
],
"total_latency_ms": 557409.335,
"total_tokens": 43094
},
"limitations": [
"Historical base-LLM and VLM checkpoints are intentionally not distributed and were not recreated in this audit.",
"Historical source revisions, dataset identities, RNG state, hardware image, and stepwise logs were not retained.",
"Current immutable pins define a future reproduction contract and are not represented as the exact historical run.",
"The English captions are translations in a bilingual report, so translation can affect judging.",
"One image-aware judge call evaluates all eight anonymous candidates per image; scores are descriptive, not a powered significance test.",
"QK-Norm and Muon change together in the improved arm, so the report does not attribute effects to Muon alone."
],
"per_image_config_scores": {
"Astronaut-Space.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"'sent to a new Earth' is not visible; no indication of a mission to a new Earth"
],
"rationale": "Mentions astronaut ('spaceman') but includes unsupported claim about being 'sent to a new Earth'.",
"visual_specificity": 1
},
"muon_from_dpo_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"spacesuit is white, not black; no glasses; astronaut is standing, not sitting; no boats or plane in image; astronaut is on the left, not far right"
],
"rationale": "Contains multiple incorrect details (color, position, invented objects) and misidentifies actions.",
"visual_specificity": 0
},
"muon_from_pretrain_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"'little boy' is not present; subject is an adult astronaut"
],
"rationale": "Incorrectly identifies subject as a little boy instead of an astronaut.",
"visual_specificity": 0
},
"muon_from_pretrain_sft": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"'museum or exhibition space' is incorrect; scene is inside a functional spacecraft, not a museum; 'objects displayed for visitors' is not visible"
],
"rationale": "Partially mentions spacecraft and instruments but incorrectly claims it's a museum exhibit.",
"visual_specificity": 2
},
"muon_from_sft_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no conversations or dialogue visible; irrelevant to image content"
],
"rationale": "No visual support for conversations; unrelated to the scene.",
"visual_specificity": 0
},
"muon_from_sft_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"subject is an astronaut, not a soldier; no other people present; no smartphone or TV remote controls visible; not a 'blue ship' but a spacecraft interior"
],
"rationale": "Completely misrepresents the scene with invented elements.",
"visual_specificity": 0
},
"without_muon_pretrained": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 3,
"material_errors": [
"astronaut is inside a spacecraft, not performing a spacewalk (no spacewalk visible)"
],
"rationale": "Correctly identifies astronaut but incorrectly claims a spacewalk; astronaut is inside the spacecraft.",
"visual_specificity": 2
},
"without_muon_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no commercial airplane; no small hill or barn; no people observing; central object is a space station, not an airplane"
],
"rationale": "Describes an invented scene with no relation to the actual image.",
"visual_specificity": 0
}
},
"Bicycle-Flowers.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"riding a bicycle (bicycle is parked)",
"bicycles everywhere (only one bicycle)",
"many cars (no cars)",
"this is my bicycle (no ownership indicated)"
],
"rationale": "Contains entirely false claims about riding, multiple bicycles, cars, and personal ownership; no accurate scene elements.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 4,
"grounding_accuracy": 4,
"hallucination_control": 3,
"material_errors": [
"two adjacent buildings (only one building visible)",
"cup placed on the ground (no cup)"
],
"rationale": "Accurately describes green parked bicycle with yellow/colorful flowers in front of a building; minor errors with extra buildings and cup.",
"visual_specificity": 4
},
"muon_from_pretrain_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"park in Paris (no park/Paris context)",
"little boy rides (no boy; bicycle is parked)",
"riverbank (no riverbank)"
],
"rationale": "Completely invented scenario with no relation to the image's bicycle, flowers, or building.",
"visual_specificity": 0
},
"muon_from_pretrain_sft": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"outdoor garden (no garden; sidewalk next to building)",
"two other bicycles (only one bicycle)",
"vases (flowers in baskets, not vases)",
"several people (no people)",
"two cars (no cars)"
],
"rationale": "Incorrectly claims garden setting, multiple bicycles, people, cars, and vases; only bicycle with flower baskets partially accurate.",
"visual_specificity": 1
},
"muon_from_sft_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"video (still image)",
"horse working in a car (no horse or car)"
],
"rationale": "Entirely unrelated to the image; mentions horse and car in a video, neither present.",
"visual_specificity": 0
},
"muon_from_sft_sft": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"bench with green tablecloth (no bench)",
"potted plant (no potted plant)",
"car parked on right (no car)",
"stop sign (no stop sign)"
],
"rationale": "Invents bench, potted plant, car, and stop sign; only green bicycle color is accurate but irrelevant to main scene.",
"visual_specificity": 1
},
"without_muon_pretrained": {
"coverage": 3,
"grounding_accuracy": 3,
"hallucination_control": 4,
"material_errors": [],
"rationale": "Accurately references the bicycle, tires/wheels, and flowers, though lacks details like green color and baskets.",
"visual_specificity": 2
},
"without_muon_sft": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"row of vases (no vases; flowers in baskets)",
"three potted plants (no potted plants)",
"two people (no people)"
],
"rationale": "Mentions bicycle and building but invents vases, potted plants, and people not present in the image.",
"visual_specificity": 1
}
},
"Chair-Elderly-Reading.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 3,
"grounding_accuracy": 5,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Correctly identifies elderly person on park bench with no hallucinations, but lacks reading activity.",
"visual_specificity": 2
},
"muon_from_dpo_sft": {
"coverage": 4,
"grounding_accuracy": 4,
"hallucination_control": 4,
"material_errors": [
"multiple benches not visible"
],
"rationale": "Correctly identifies elderly man on park bench holding a book, trees in background; only error is multiple benches.",
"visual_specificity": 4
},
"muon_from_pretrain_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"young man instead of elderly",
"bed instead of park bench"
],
"rationale": "No young man or bed visible; subject and setting are entirely incorrect.",
"visual_specificity": 0
},
"muon_from_pretrain_sft": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"library instead of park",
"table instead of bench",
"cluttered bookshelves not present",
"potted plant not visible"
],
"rationale": "Incorrect library setting; no table, bookshelves, or potted plant; only elderly man with glasses is correct.",
"visual_specificity": 1
},
"muon_from_sft_pretrained": {
"coverage": 3,
"grounding_accuracy": 5,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Correctly identifies elderly person on park bench with no hallucinations, but lacks reading activity.",
"visual_specificity": 1
},
"muon_from_sft_sft": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"reading newspaper instead of book",
"multiple cars not visible",
"person at top of frame not present"
],
"rationale": "Reads book (not newspaper); no cars or person at top; bench and park are correct but other details invented.",
"visual_specificity": 1
},
"without_muon_pretrained": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 4,
"material_errors": [
"incorrect gender (woman instead of man)"
],
"rationale": "Correct park setting and reading a book, but misidentifies gender as woman.",
"visual_specificity": 2
},
"without_muon_sft": {
"coverage": 3,
"grounding_accuracy": 3,
"hallucination_control": 2,
"material_errors": [
"several cars parked nearby not visible",
"another bench in the background not present"
],
"rationale": "Correct elderly man with glasses reading a book on park bench, but invents cars and another bench.",
"visual_specificity": 3
}
},
"Dog-Woman-Sea.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no ball, palm, or throwing action in image"
],
"rationale": "Completely irrelevant; no ball or throwing present.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 1,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"woman sits on sand, not a bench",
"woman not holding dog; dog sits beside her",
"woman wears sleeveless top/jeans, not a dress",
"no other people in background"
],
"rationale": "Contains multiple hallucinations: bench, held dog, dress, and other people, all absent.",
"visual_specificity": 0
},
"muon_from_pretrain_pretrained": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 4,
"material_errors": [
"dog is sitting, not walking",
"missing woman sitting beside dog"
],
"rationale": "Correct about dog and seaside but misses woman and incorrectly states dog is walking.",
"visual_specificity": 1
},
"muon_from_pretrain_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no beach bench; woman sits on sand",
"woman not holding dog; dog sits beside her",
"dog is adult, not a puppy",
"no other people, handbags, chairs, or bench"
],
"rationale": "Filled with hallucinated objects (bench, handbags, chairs) and incorrect actions (holding puppy).",
"visual_specificity": 0
},
"muon_from_sft_pretrained": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 3,
"material_errors": [
"woman wears light blue sleeveless top and blue jeans, not white long dress"
],
"rationale": "Identifies woman and dog but misrepresents clothing (white long dress vs. blue top/jeans).",
"visual_specificity": 1
},
"muon_from_sft_sft": {
"coverage": 2,
"grounding_accuracy": 1,
"hallucination_control": 1,
"material_errors": [
"woman not holding dog; dog sits beside her",
"no multiple figures in background",
"dog is brown and white, not just brown"
],
"rationale": "Incorrectly claims woman holds dog and background has multiple people; dog color misrepresented.",
"visual_specificity": 1
},
"without_muon_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no young person decorating beach",
"no dog's head; whole dog sits beside woman"
],
"rationale": "Entirely misrepresents scene; no decorating or dog's head present.",
"visual_specificity": 0
},
"without_muon_sft": {
"coverage": 3,
"grounding_accuracy": 4,
"hallucination_control": 4,
"material_errors": [
"dog is not on a blue and white checkered blanket; dog sits on sand"
],
"rationale": "Correctly identifies woman sitting on beach with dog beside her; falsely claims dog sits on a checkered blanket (not present).",
"visual_specificity": 3
}
},
"Panda-Grassland.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"subject is a black-and-white panda, not a white bear"
],
"rationale": "Refers to the panda as a 'white bear'; panda has distinct black-and-white fur.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 4,
"grounding_accuracy": 3,
"hallucination_control": 2,
"material_errors": [
"panda is lying, not standing; no evidence of sun warmth"
],
"rationale": "Accurately identifies giant panda with black and white markings on lush green grassland but incorrectly states it is standing and mentions unevidenced sun warmth.",
"visual_specificity": 4
},
"muon_from_pretrain_pretrained": {
"coverage": 3,
"grounding_accuracy": 4,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Accurately describes the panda as a bear with cute black-and-white fur, with no incorrect details.",
"visual_specificity": 3
},
"muon_from_pretrain_sft": {
"coverage": 3,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"panda is lying, not sitting; no flowers visible; no grove (only grass)"
],
"rationale": "Identifies black-and-white bear on green grass but includes hallucinations (flowers, grove) and incorrect position (sitting).",
"visual_specificity": 2
},
"muon_from_sft_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"subject is panda, not zebra; setting is grass, not woods"
],
"rationale": "Incorrectly identifies subject as zebra and setting as woods; image shows a panda on grass.",
"visual_specificity": 0
},
"muon_from_sft_sft": {
"coverage": 3,
"grounding_accuracy": 2,
"hallucination_control": 3,
"material_errors": [
"panda is lying, not standing"
],
"rationale": "Identifies black-and-white panda and grass but incorrectly states the panda is standing (it is lying).",
"visual_specificity": 2
},
"without_muon_pretrained": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"no evidence of zoo; panda is not eating bamboo (lying on grass)"
],
"rationale": "States panda is in a zoo eating bamboo; image shows panda lying on grass with no bamboo or zoo elements.",
"visual_specificity": 0
},
"without_muon_sft": {
"coverage": 3,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"panda is not wearing glasses; no evidence of sun or rain; conflicting positions (sitting vs lying)"
],
"rationale": "Identifies black-and-white panda on grass and mentions lying/resting but includes hallucinations (glasses, sun/rain) and conflicting position (sitting).",
"visual_specificity": 1
}
},
"Rainbow-Falls.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Vaguely refers to a 'water landscape' but lacks specific details about the waterfall or rainbow.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"fountain is not present (it is a waterfall)",
"umbrella on water is not present",
"visitors are not present"
],
"rationale": "Incorrectly identifies the waterfall as a 'fountain' and adds non-existent elements like umbrella and visitors.",
"visual_specificity": 0
},
"muon_from_pretrain_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"cave is not present",
"water source is a waterfall, not droplets from a cave"
],
"rationale": "Invents a 'cave' and misrepresents the water source; does not depict the waterfall.",
"visual_specificity": 0
},
"muon_from_pretrain_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"rough seas not present",
"cloudy sky not present (sky is clear)",
"mountain with white dome not present",
"people not present"
],
"rationale": "Describes an unrelated seascape with people and mountains, not the waterfall scene.",
"visual_specificity": 0
},
"muon_from_sft_pretrained": {
"coverage": 2,
"grounding_accuracy": 3,
"hallucination_control": 3,
"material_errors": [
"missing mention of waterfall and rainbow",
"perspective as 'from the mountaintop' is unclear"
],
"rationale": "Accurately notes water glistening in sunlight but omits key elements (waterfall, rainbow) and has unclear perspective.",
"visual_specificity": 2
},
"muon_from_sft_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"waterfall is misidentified as a fountain"
],
"rationale": "Falsely labels the waterfall as a 'fountain landscape' with no basis in the image.",
"visual_specificity": 1
},
"without_muon_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"rainbow bridge is not present",
"focus on water droplets is incorrect"
],
"rationale": "Mentions non-existent 'rainbow bridge' and misfocuses on droplets; fails to describe the waterfall.",
"visual_specificity": 0
},
"without_muon_sft": {
"coverage": 3,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"highway is not present",
"giant rainbow flag is not present"
],
"rationale": "Correctly identifies a large waterfall but includes hallucinated elements (highway, rainbow flag) not in the image.",
"visual_specificity": 2
}
},
"city-traffic.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 1,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no elevators visible; main subject (heavy traffic) not mentioned"
],
"rationale": "Irrelevant focus on elevators and sidewalks; misses the prominent heavy traffic and tall buildings.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 2,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no parked cars (all cars are in traffic); no traffic lights visible; no bus present"
],
"rationale": "Falsely claims parked cars, traffic lights, and a bus; these elements are not visible in the image.",
"visual_specificity": 2
},
"muon_from_pretrain_pretrained": {
"coverage": 1,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no bus visible in the image"
],
"rationale": "A tall building is present, but there is no bus. The main subject is heavy traffic, not a bus traveling to a building.",
"visual_specificity": 1
},
"muon_from_pretrain_sft": {
"coverage": 4,
"grounding_accuracy": 3,
"hallucination_control": 2,
"material_errors": [
"no motorcycles visible; no passersby (pedestrians) visible"
],
"rationale": "Correctly identifies busy street, traffic, tall buildings, but falsely includes motorcycles and passersby.",
"visual_specificity": 3
},
"muon_from_sft_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"mentions 'monitoring' which is not visible; traffic lights are not the main subject and not clearly present"
],
"rationale": "The image depicts a busy nighttime city street with traffic and buildings, not monitoring of traffic lights. No monitoring activity or distinct traffic lights are visible.",
"visual_specificity": 0
},
"muon_from_sft_sft": {
"coverage": 5,
"grounding_accuracy": 5,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Accurately describes the nighttime busy city street with cars, trucks, high-rise buildings, and bustling traffic (stationary and moving) without hallucinations.",
"visual_specificity": 4
},
"without_muon_pretrained": {
"coverage": 1,
"grounding_accuracy": 2,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Generic mention of 'city streets at nightfall' is accurate but lacks specific details about traffic, buildings, or activity.",
"visual_specificity": 0
},
"without_muon_sft": {
"coverage": 4,
"grounding_accuracy": 3,
"hallucination_control": 2,
"material_errors": [
"no pedestrians visible; no traffic lights clearly visible"
],
"rationale": "Correctly identifies busy street, traffic, cars, truck, tall buildings, and streetlights, but falsely mentions pedestrians and traffic lights which are not present.",
"visual_specificity": 3
}
},
"dance.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 1,
"material_errors": [
"actors (image shows a dancer, not actors)"
],
"rationale": "Incorrectly identifies the subject as 'actors' instead of a dancer; no other accurate details.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"black dress (actual: light-colored dress)",
"several people watching (no people present)",
"mobile phones (no phones in image)"
],
"rationale": "Incorrectly describes the dress as black (it is light-colored) and invents non-existent people and mobile phones.",
"visual_specificity": 0
},
"muon_from_pretrain_pretrained": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 2,
"material_errors": [
"performers (only one dancer)",
"colorful costumes (dress is light-colored, not colorful)"
],
"rationale": "Vaguely mentions a performance but incorrectly refers to multiple 'performers' and 'colorful costumes' (image has one dancer in a light dress).",
"visual_specificity": 0
},
"muon_from_pretrain_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"walking down walkway (dancing on stage)",
"holding umbrella (no umbrella)",
"chairs along walkway (no chairs)",
"kites (no kites)"
],
"rationale": "Contains multiple hallucinations: walkway, umbrella, chairs, and kites are all absent; the subject is dancing on stage, not walking.",
"visual_specificity": 0
},
"muon_from_sft_pretrained": {
"coverage": 4,
"grounding_accuracy": 5,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Accurately identifies a dancer performing on stage in a stylish outfit, with no false claims; aligns with visible elements (single dancer, stage, elegant dress).",
"visual_specificity": 3
},
"muon_from_sft_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"tuxedo (wearing a dress)",
"holding microphone (no microphone)",
"several people around (no people)",
"wine glass and bottles (no such items)"
],
"rationale": "Contains multiple false claims: tuxedo (dress), microphone, people, and wine glasses/bottles are all absent.",
"visual_specificity": 0
},
"without_muon_pretrained": {
"coverage": 3,
"grounding_accuracy": 5,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Correctly states a dancer is performing on stage but lacks visual details (e.g., outfit description).",
"visual_specificity": 1
},
"without_muon_sft": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"dance steps soaring high (dancer is on stage floor)",
"several chairs on stage (no chairs)",
"clock on stage (no clock)"
],
"rationale": "Mentions a dance on stage but invents 'soaring steps' (dancer is grounded) and non-existent chairs/clock.",
"visual_specificity": 1
}
}
},
"ranking_by_overall": [
"without_muon_sft",
"without_muon_pretrained",
"muon_from_dpo_sft",
"muon_from_sft_pretrained",
"muon_from_sft_sft",
"muon_from_pretrain_sft",
"muon_from_dpo_pretrained",
"muon_from_pretrain_pretrained"
],
"retained": {
"cells": 8,
"images": 8,
"outputs": 64
},
"schema_version": "exp8-4-summary-v1",
"scientific_findings": {
"author_claims_are_historical_observations_not_acceptance_gates": true,
"muon_only_causal_claim_avoided": true,
"sft_minus_pretrained_average": 0.1718,
"top_configuration": "without_muon_sft",
"top_configuration_overall": 1.9062
},
"stage_averages": {
"pretrained": {
"coverage": 1.125,
"grounding_accuracy": 1.375,
"hallucination_control": 1.875,
"overall": 1.2656,
"visual_specificity": 0.6875
},
"sft": {
"coverage": 1.7812,
"grounding_accuracy": 1.4688,
"hallucination_control": 1.0312,
"overall": 1.4374,
"visual_specificity": 1.4688
}
},
"status": "passed"
}
chapter8/MiniMind-pretrain/validation/test_training_report_audit.py
+135
View File
@@ -0,0 +1,135 @@
from __future__ import annotations
import hashlib
import json
import shutil
from pathlib import Path
import pytest
import run_training_report_audit as audit
import validate_evidence as validator
def digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def write_json(path: Path, value: object) -> None:
path.write_text(
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
def test_historical_report_parser_retains_complete_six_cell_matrix() -> None:
retained = audit.parse_retained_outputs()
assert retained["cell_count"] == 6
assert retained["output_count"] == 49
assert {
(cell["arm"], cell["stage"]): cell["pair_count"]
for cell in retained["cells"]
} == audit.EXPECTED_COUNTS
comparisons = audit.selected_comparisons(retained)
assert len(comparisons) == 8
assert {row["stage"] for row in comparisons} == set(audit.STAGES)
assert all(set(row["arms"]) == set(audit.ARMS) for row in comparisons)
def test_judge_requests_are_arm_blind_and_bound_to_exact_outputs() -> None:
retained = audit.parse_retained_outputs()
for comparison in audit.selected_comparisons(retained):
mapping = audit.blind_mapping(comparison["case_id"])
payload = audit.judge_payload(comparison, mapping, "judge-model")
serialized = json.dumps(payload, ensure_ascii=False).lower()
assert "qk_norm_muon" not in serialized
assert '"original"' not in serialized
user_payload = json.loads(payload["messages"][1]["content"])
for label, arm in mapping.items():
assert user_payload["candidates"][label]["historical_output"] == comparison["arms"][arm]["output"]
def make_validation_copy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, Path]:
latest = json.loads(audit.LATEST_PATH.read_text(encoding="utf-8"))
canonical_run = audit.EXPERIMENT_DIR / latest["run_dir"]
temp_repo = tmp_path / "repo"
temp_experiment = temp_repo / "chapter8/MiniMind-pretrain"
temp_run = temp_experiment / latest["run_dir"]
temp_run.parent.mkdir(parents=True)
shutil.copytree(canonical_run, temp_run)
manifest = json.loads((canonical_run / "manifest.json").read_text(encoding="utf-8"))
for record in manifest["inputs"]:
source = audit.REPO_ROOT / record["path"]
destination = temp_repo / record["path"]
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, destination)
temp_latest = temp_experiment / "validation/latest.json"
temp_latest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(audit.LATEST_PATH, temp_latest)
monkeypatch.setattr(validator, "REPO_ROOT", temp_repo)
monkeypatch.setattr(validator, "EXPERIMENT_DIR", temp_experiment)
return temp_latest, temp_run
def refresh_outer_hashes(latest_path: Path, run_dir: Path, artifact_name: str) -> None:
manifest_path = run_dir / "manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
artifact_path = run_dir / artifact_name
record = next(record for record in manifest["artifacts"] if record["path"] == artifact_name)
record["bytes"] = artifact_path.stat().st_size
record["sha256"] = digest(artifact_path)
write_json(manifest_path, manifest)
latest = json.loads(latest_path.read_text(encoding="utf-8"))
latest["manifest_sha256"] = digest(manifest_path)
write_json(latest_path, latest)
def test_canonical_evidence_passes_fail_closed_validator() -> None:
result = validator.validate()
assert result["status"] == "passed"
assert result["outputs_verified"] == 49
assert result["judge_receipts_verified"] == 8
def test_validator_rejects_raw_response_normalization_tamper(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
latest_path, run_dir = make_validation_copy(tmp_path, monkeypatch)
receipts_path = run_dir / "judge_receipts.json"
receipts = json.loads(receipts_path.read_text(encoding="utf-8"))
response = receipts["calls"][0]["response"]
raw_judgment = json.loads(response["choices"][0]["message"]["content"])
raw_judgment["winner"] = "tie" if raw_judgment["winner"] != "tie" else "A"
response["choices"][0]["message"]["content"] = json.dumps(raw_judgment)
write_json(receipts_path, receipts)
refresh_outer_hashes(latest_path, run_dir, "judge_receipts.json")
with pytest.raises(AssertionError, match="normalized judgment"):
validator.validate(latest_path)
def test_validator_rejects_retained_output_request_binding_tamper(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
latest_path, run_dir = make_validation_copy(tmp_path, monkeypatch)
retained_path = run_dir / "retained_outputs.json"
retained = json.loads(retained_path.read_text(encoding="utf-8"))
retained["cells"][0]["pairs"][3]["output"] += " altered"
write_json(retained_path, retained)
refresh_outer_hashes(latest_path, run_dir, "retained_outputs.json")
with pytest.raises(AssertionError, match="not bound to the retained output"):
validator.validate(latest_path)
def test_validator_rejects_frozen_dataset_revision_tamper(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
latest_path, run_dir = make_validation_copy(tmp_path, monkeypatch)
contract_path = run_dir / "reproduction_contract.json"
contract = json.loads(contract_path.read_text(encoding="utf-8"))
contract["future_reproduction"]["dataset"]["revision"] = "0" * 40
write_json(contract_path, contract)
refresh_outer_hashes(latest_path, run_dir, "reproduction_contract.json")
with pytest.raises(AssertionError, match="dataset revision mismatch"):
validator.validate(latest_path)
chapter8/MiniMind-pretrain/validation/test_vlm_training_report_audit.py
+161
View File
@@ -0,0 +1,161 @@
import hashlib
import json
import shutil
import sys
from pathlib import Path
import pytest
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import run_vlm_training_report_audit as audit
import validate_vlm_evidence as validator
RUN_DIR = HERE / "runs" / audit.DEFAULT_RUN_ID
def write_json(path: Path, value):
path.write_text(
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
def reseal_artifact(run_dir: Path, name: str):
manifest_path = run_dir / "manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
artifact = next(row for row in manifest["artifacts"] if row["path"] == name)
artifact["bytes"] = (run_dir / name).stat().st_size
artifact["sha256"] = hashlib.sha256((run_dir / name).read_bytes()).hexdigest()
write_json(manifest_path, manifest)
def copied_run(tmp_path: Path) -> Path:
target = tmp_path / "run"
shutil.copytree(RUN_DIR, target)
return target
def test_parser_retains_all_eight_cells_and_64_outputs():
retained = audit.parse_retained_outputs()
assert retained["cell_count"] == 8
assert retained["output_count"] == 64
assert tuple(cell["config"] for cell in retained["cells"]) == audit.CONFIGS
assert all(cell["output_count"] == 8 for cell in retained["cells"])
assert all(
{row["image"] for row in cell["outputs"]} == set(audit.IMAGE_FILES)
for cell in retained["cells"]
)
def test_blind_mapping_is_deterministic_bijective_and_image_specific():
mappings = [audit.blind_mapping(image) for image in audit.IMAGE_FILES]
assert all(set(mapping) == set(audit.LABELS) for mapping in mappings)
assert all(set(mapping.values()) == set(audit.CONFIGS) for mapping in mappings)
assert all(
mapping == audit.blind_mapping(image)
for image, mapping in zip(audit.IMAGE_FILES, mappings, strict=True)
)
assert len({tuple(mapping.items()) for mapping in mappings}) > 1
def test_reproduction_contract_separates_original_and_improved_sources():
contract = audit.reproduction_contract()
vlm = contract["future_reproduction"]["vlm_source"]
commands = contract["future_reproduction"]["commands"]
assert vlm["original_revision"] == audit.ORIGINAL_VLM_REVISION
assert vlm["qk_norm_muon_revision"] == audit.IMPROVED_VLM_REVISION
assert vlm["original_revision"] != vlm["qk_norm_muon_revision"]
assert audit.ORIGINAL_VLM_REVISION in commands["original_source"]
assert audit.IMPROVED_VLM_REVISION in commands["improved_source"]
assert "checkout --detach" in commands["original_source"]
assert "checkout --detach" in commands["improved_source"]
assert (
contract["historical_evidence_boundary"]["historical_vlm_checkpoint_hashes_retained"]
is False
)
assert contract["checkpoint_policy"]["acceptance_artifact"] is False
def test_canonical_evidence_passes_fail_closed_validator():
result = validator.validate_run(RUN_DIR)
assert result["status"] == "passed"
assert result["cells"] == 8
assert result["outputs"] == 64
assert result["judge_receipts"] == 8
def test_vlm_latest_pointer_does_not_overwrite_experiment_8_3():
assert audit.LATEST_PATH.name == "latest_vlm.json"
vlm_latest = json.loads(audit.LATEST_PATH.read_text(encoding="utf-8"))
llm_latest = json.loads((HERE / "latest.json").read_text(encoding="utf-8"))
assert vlm_latest["experiment"] == "8-4"
assert llm_latest["experiment"] == "8-3"
def test_receipts_are_real_image_aware_unique_and_arm_blind():
receipts = json.loads((RUN_DIR / "judge_receipts.json").read_text(encoding="utf-8"))["calls"]
assert len({row["response_id"] for row in receipts}) == 8
retained = audit.parse_retained_outputs()
for receipt in receipts:
assert receipt["http_status"] == 200
assert receipt["usage"]["total_tokens"] > 0
content = receipt["request"]["messages"][1]["content"]
assert content[0]["type"] == "image_url"
assert content[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
prompt = json.loads(content[1]["text"])
expected = audit.outputs_for_image(retained, receipt["image"])
assert prompt["candidates"] == {
label: expected[config] for label, config in receipt["blind_map"].items()
}
assert not any(config in content[1]["text"] for config in audit.CONFIGS)
def test_tampered_retained_output_fails_even_after_resealing(tmp_path):
run_dir = copied_run(tmp_path)
path = run_dir / "retained_outputs.json"
data = json.loads(path.read_text(encoding="utf-8"))
data["cells"][0]["outputs"][0]["output"] += " tampered"
write_json(path, data)
reseal_artifact(run_dir, path.name)
with pytest.raises(validator.EvidenceError, match="retained outputs"):
validator.validate_run(run_dir, verify_latest=False)
def test_tampered_request_image_fails_even_after_resealing(tmp_path):
run_dir = copied_run(tmp_path)
path = run_dir / "judge_receipts.json"
data = json.loads(path.read_text(encoding="utf-8"))
url = data["calls"][0]["request"]["messages"][1]["content"][0]["image_url"]["url"]
prefix, encoded = url.split(",", 1)
replacement = "A" if encoded[-2] != "A" else "B"
data["calls"][0]["request"]["messages"][1]["content"][0]["image_url"]["url"] = (
prefix + "," + encoded[:-2] + replacement + encoded[-1]
)
write_json(path, data)
reseal_artifact(run_dir, path.name)
with pytest.raises(validator.EvidenceError, match="request image bytes"):
validator.validate_run(run_dir, verify_latest=False)
def test_normalized_judgment_must_match_raw_provider_response(tmp_path):
run_dir = copied_run(tmp_path)
path = run_dir / "judge_receipts.json"
data = json.loads(path.read_text(encoding="utf-8"))
current = data["calls"][0]["judgment"]["candidates"]["A"]["grounding_accuracy"]
data["calls"][0]["judgment"]["candidates"]["A"]["grounding_accuracy"] = 0 if current != 0 else 1
write_json(path, data)
reseal_artifact(run_dir, path.name)
with pytest.raises(validator.EvidenceError, match="not derived from raw response"):
validator.validate_run(run_dir, verify_latest=False)
def test_reproduction_pin_tampering_fails_even_after_resealing(tmp_path):
run_dir = copied_run(tmp_path)
path = run_dir / "reproduction_contract.json"
data = json.loads(path.read_text(encoding="utf-8"))
data["future_reproduction"]["vlm_source"]["original_revision"] = "0" * 40
write_json(path, data)
reseal_artifact(run_dir, path.name)
with pytest.raises(validator.EvidenceError, match="reproduction contract"):
validator.validate_run(run_dir, verify_latest=False)
chapter8/MiniMind-pretrain/validation/validate_evidence.py
+313
View File
@@ -0,0 +1,313 @@
#!/usr/bin/env python3
"""Fail-closed validator for Experiment 8-3 retained training evidence."""
from __future__ import annotations
import argparse
import hashlib
import json
import re
from pathlib import Path
from typing import Any
HERE = Path(__file__).resolve().parent
EXPERIMENT_DIR = HERE.parent
REPO_ROOT = EXPERIMENT_DIR.parents[1]
LATEST_PATH = HERE / "latest.json"
ARMS = {"original", "qk_norm_muon"}
STAGES = {"pretrain", "sft", "dpo"}
EXPECTED_COUNTS = {
("original", "pretrain"): 7,
("original", "sft"): 8,
("original", "dpo"): 9,
("qk_norm_muon", "pretrain"): 7,
("qk_norm_muon", "sft"): 9,
("qk_norm_muon", "dpo"): 9,
}
EXPECTED_SOURCE_REVISION = "8bdc5d97d5845a8c1ac2ed56a5b8b4c0d0fb0795"
EXPECTED_DATASET_REVISION = "84983ed4dec7836d240577760c1d6be5d4cabcf9"
EXPECTED_DATASET_FILES = {
"pretrain_hq.jsonl": (
"9801b0d2210c61c2e4bc130f6dc4b3c870698a88d04af8f103c23dd5f0ce2440",
1_669_750_047,
),
"sft_512.jsonl": (
"053b7d09574e48a86232e929211434ff9e5016c6ed13312e63687dd52edcbebf",
7_531_517_862,
),
"dpo.jsonl": (
"ee934a8a455ccc99d1334d63e1254dd1d64f497fd067cfcbb71e3043f5b46768",
53_653_322,
),
}
SECRET_PATTERNS = (
re.compile(r"(?i)authorization\s*[:=]\s*bearer\s+\S+"),
re.compile(r"(?i)(?:api[_-]?key|secret)\s*[:=]\s*[A-Za-z0-9._-]{16,}"),
re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b"),
)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def load_json(path: Path) -> dict[str, Any]:
value = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(value, dict):
raise AssertionError(f"{path} must contain a JSON object")
return value
def resolve_relative(base: Path, relative: str) -> Path:
path = (base / relative).resolve()
if not path.is_relative_to(base.resolve()):
raise AssertionError(f"path escapes evidence root: {relative}")
return path
def check_record(path: Path, record: dict[str, Any]) -> None:
if not path.is_file():
raise AssertionError(f"missing declared file: {path}")
if path.stat().st_size != record.get("bytes"):
raise AssertionError(f"byte count mismatch: {path}")
if sha256_file(path) != record.get("sha256"):
raise AssertionError(f"SHA-256 mismatch: {path}")
def parse_response_content(content: str) -> dict[str, Any]:
stripped = content.strip()
if stripped.startswith("```"):
stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
stripped = re.sub(r"\s*```$", "", stripped)
value = json.loads(stripped)
if not isinstance(value, dict):
raise AssertionError("judge response content must decode to an object")
return value
def find_pair(
retained: dict[str, Any], arm: str, stage: str, keyword: str
) -> dict[str, str]:
cells = [
cell
for cell in retained["cells"]
if cell.get("arm") == arm and cell.get("stage") == stage
]
if len(cells) != 1:
raise AssertionError(f"missing or duplicate retained cell: {arm}/{stage}")
matches = [
pair
for pair in cells[0]["pairs"]
if keyword.lower() in pair.get("prompt", "").lower()
]
if len(matches) != 1:
raise AssertionError(f"missing or duplicate selected prompt: {arm}/{stage}/{keyword}")
return matches[0]
def validate(latest_path: Path = LATEST_PATH) -> dict[str, Any]:
latest = load_json(latest_path)
if latest.get("experiment") != "8-3" or latest.get("status") != "passed":
raise AssertionError("latest pointer is not a passed Experiment 8-3 run")
run_dir = resolve_relative(EXPERIMENT_DIR, latest["run_dir"])
manifest_path = run_dir / "manifest.json"
if sha256_file(manifest_path) != latest.get("manifest_sha256"):
raise AssertionError("latest manifest hash mismatch")
manifest = load_json(manifest_path)
if manifest.get("run_id") != latest.get("run_id"):
raise AssertionError("run ID mismatch between latest and manifest")
if manifest.get("experiment") != "8-3" or manifest.get("status") != "passed":
raise AssertionError("manifest is not a passed Experiment 8-3 run")
if manifest.get("checkpoint_policy") != "not distributed; not an acceptance artifact":
raise AssertionError("manifest checkpoint policy is incorrect")
inputs = manifest.get("inputs")
artifacts = manifest.get("artifacts")
if not isinstance(inputs, list) or len(inputs) != 5:
raise AssertionError("manifest must bind exactly five repository inputs")
if not isinstance(artifacts, list) or len(artifacts) != 5:
raise AssertionError("manifest must bind exactly five run artifacts")
for record in inputs:
check_record(resolve_relative(REPO_ROOT, record["path"]), record)
for record in artifacts:
check_record(resolve_relative(run_dir, record["path"]), record)
retained = load_json(run_dir / "retained_outputs.json")
report_record = next(
(record for record in inputs if record.get("path") == retained.get("source_report")),
None,
)
if report_record is None or retained.get("source_report_sha256") != report_record.get("sha256"):
raise AssertionError("retained source-report hash does not match the manifest input")
if retained.get("cell_count") != 6 or retained.get("output_count") != 49:
raise AssertionError("retained report must contain six cells and 49 outputs")
if set(retained.get("arms", [])) != ARMS or set(retained.get("stages", [])) != STAGES:
raise AssertionError("retained report arm/stage coverage is incomplete")
cells = retained.get("cells")
if not isinstance(cells, list) or len(cells) != 6:
raise AssertionError("retained cells are malformed")
combos: set[tuple[str, str]] = set()
for cell in cells:
combo = (cell.get("arm"), cell.get("stage"))
if combo in combos or combo not in EXPECTED_COUNTS:
raise AssertionError(f"duplicate or unexpected cell: {combo}")
combos.add(combo)
pairs = cell.get("pairs")
if not isinstance(pairs, list) or len(pairs) != EXPECTED_COUNTS[combo]:
raise AssertionError(f"wrong retained pair count for {combo}")
if any(not pair.get("prompt") or not pair.get("output") for pair in pairs):
raise AssertionError(f"empty retained prompt/output in {combo}")
if combos != set(EXPECTED_COUNTS):
raise AssertionError("not all arm/stage cells are present")
receipts_root = load_json(run_dir / "judge_receipts.json")
if receipts_root.get("credential_headers_retained") is not False:
raise AssertionError("credential header retention must be explicitly false")
calls = receipts_root.get("calls")
if not isinstance(calls, list) or len(calls) != 8:
raise AssertionError("exactly eight raw judge calls are required")
response_ids: set[str] = set()
normalized_rows: dict[str, Any] = {}
for expected_case_id, call in enumerate(calls, start=1):
if call.get("case_id") != expected_case_id or call.get("http_status") != 200:
raise AssertionError("judge calls must be successful and ordered by case ID")
if call.get("credential_headers_retained") is not False:
raise AssertionError("per-call credential retention boundary is missing")
response_id = call.get("response_id")
if not isinstance(response_id, str) or not response_id or response_id in response_ids:
raise AssertionError("judge response IDs must be present and unique")
response_ids.add(response_id)
if call.get("latency_ms", 0) <= 0 or call.get("usage", {}).get("total_tokens", 0) <= 0:
raise AssertionError("judge usage and positive latency must be retained")
raw_response = call.get("response", {})
if raw_response.get("id") != response_id or raw_response.get("usage") != call.get("usage"):
raise AssertionError("copied response ID/usage does not match raw response")
try:
content = raw_response["choices"][0]["message"]["content"]
except (KeyError, IndexError, TypeError) as exc:
raise AssertionError("raw judge response is missing message content") from exc
judgment = parse_response_content(content)
if judgment != call.get("judgment"):
raise AssertionError("normalized judgment does not match raw response content")
if str(judgment.get("case_id")) != str(expected_case_id):
raise AssertionError("raw judgment has the wrong case ID")
if set(judgment.get("candidates", {})) != {"A", "B"}:
raise AssertionError("judge judgment must score A and B")
if judgment.get("winner") not in {"A", "B", "tie"}:
raise AssertionError("judge winner is invalid")
mapping = call.get("blind_map")
if not isinstance(mapping, dict) or set(mapping) != {"A", "B"} or set(mapping.values()) != ARMS:
raise AssertionError("blind mapping must cover both arms")
request = call.get("request")
request_text = json.dumps(request, ensure_ascii=False).lower()
if "qk_norm_muon" in request_text or '"original"' in request_text:
raise AssertionError("judge request leaks an arm identity")
try:
user_payload = json.loads(request["messages"][1]["content"])
request_candidates = user_payload["candidates"]
except (KeyError, IndexError, TypeError, json.JSONDecodeError) as exc:
raise AssertionError("judge request is missing structured candidates") from exc
if user_payload.get("case_id") != expected_case_id:
raise AssertionError("judge request case ID mismatch")
for label, arm in mapping.items():
retained_pair = find_pair(retained, arm, call["stage"], call["keyword"])
expected_candidate = {
"historical_prompt": retained_pair["prompt"],
"historical_output": retained_pair["output"],
}
if request_candidates.get(label) != expected_candidate:
raise AssertionError("raw judge request is not bound to the retained output")
normalized_rows[str(expected_case_id)] = {
mapping[label]: score for label, score in judgment["candidates"].items()
}
contract = load_json(run_dir / "reproduction_contract.json")
future = contract.get("future_reproduction", {})
source = future.get("source", {})
dataset = future.get("dataset", {})
if source.get("revision") != EXPECTED_SOURCE_REVISION:
raise AssertionError("frozen MiniMind source revision mismatch")
source_hashes = source.get("file_sha256")
if not isinstance(source_hashes, dict) or len(source_hashes) < 12:
raise AssertionError("frozen source file hashes are incomplete")
if any(not re.fullmatch(r"[0-9a-f]{64}", value) for value in source_hashes.values()):
raise AssertionError("invalid frozen source SHA-256")
if dataset.get("revision") != EXPECTED_DATASET_REVISION:
raise AssertionError("frozen dataset revision mismatch")
dataset_files = dataset.get("files", {})
for name, (expected_hash, expected_bytes) in EXPECTED_DATASET_FILES.items():
record = dataset_files.get(name, {})
if record.get("lfs_sha256") != expected_hash or record.get("bytes") != expected_bytes:
raise AssertionError(f"frozen dataset file mismatch: {name}")
if len(future.get("commands", {})) != 6:
raise AssertionError("all six reproduction commands are required")
boundary = contract.get("historical_evidence_boundary", {})
for key in (
"historical_source_revision_retained",
"historical_dataset_hashes_retained",
"historical_checkpoint_hashes_retained",
"historical_stepwise_training_logs_retained",
):
if boundary.get(key) is not False:
raise AssertionError(f"historical provenance boundary is not explicit: {key}")
mechanisms = contract.get("model_and_training", {}).get("source_verified_mechanisms", {})
if not mechanisms or not all(value is True for value in mechanisms.values()):
raise AssertionError("source mechanism assertions are incomplete")
policy = contract.get("checkpoint_policy", {})
if policy.get("distributed_with_book") is not False or policy.get("acceptance_artifact") is not False:
raise AssertionError("checkpoint policy does not match the book contract")
summary = load_json(run_dir / "summary.json")
if summary.get("status") != "passed" or summary.get("acceptance", {}).get("passed") is not True:
raise AssertionError("summary acceptance did not pass")
acceptance = summary["acceptance"]
if not all(value is True for key, value in acceptance.items() if key != "passed"):
failed = [key for key, value in acceptance.items() if key != "passed" and value is not True]
raise AssertionError(f"required acceptance gates failed: {failed}")
if summary.get("per_case_arm_scores") != normalized_rows:
raise AssertionError("summary scores do not match raw judge responses")
findings = summary.get("scientific_findings", {})
if not isinstance(findings.get("blind_judge_prefers_qk_norm_muon_overall"), bool):
raise AssertionError("comparative scientific finding is missing")
if findings.get("reported_loss_comparison_retained_but_not_independently_recomputed") is not True:
raise AssertionError("loss-evidence qualification is missing")
for record in artifacts:
path = resolve_relative(run_dir, record["path"])
if path.suffix not in {".json", ".md"}:
continue
text = path.read_text(encoding="utf-8")
for pattern in SECRET_PATTERNS:
if pattern.search(text):
raise AssertionError(f"possible credential in retained artifact: {path.name}")
return {
"experiment": "8-3",
"run_id": latest["run_id"],
"status": "passed",
"inputs_verified": len(inputs),
"artifacts_verified": len(artifacts),
"judge_receipts_verified": len(calls),
"outputs_verified": retained["output_count"],
"manifest_sha256": latest["manifest_sha256"],
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--latest", type=Path, default=LATEST_PATH)
args = parser.parse_args()
result = validate(args.latest.resolve())
print(json.dumps(result, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
chapter8/MiniMind-pretrain/validation/validate_vlm_evidence.py
Executable
+359
View File
@@ -0,0 +1,359 @@
#!/usr/bin/env python3
"""Fail-closed validator for the canonical Experiment 8-4 evidence package."""
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import re
from pathlib import Path
from typing import Any
import run_vlm_training_report_audit as audit
HERE = Path(__file__).resolve().parent
EXPERIMENT_DIR = HERE.parent
REPO_ROOT = EXPERIMENT_DIR.parents[1]
EXPECTED_ARTIFACTS = {
"retained_outputs.json",
"reproduction_contract.json",
"judge_receipts.json",
"summary.json",
"report.md",
}
FORBIDDEN_SECRET_PATTERNS = (
re.compile(r"(?i)authorization\s*[:=]\s*bearer"),
re.compile(r"(?i)api[_-]?key\s*[:=]\s*['\"]?[A-Za-z0-9_-]{16,}"),
re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b"),
)
class EvidenceError(RuntimeError):
pass
def fail(message: str) -> None:
raise EvidenceError(message)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def load_json(path: Path) -> Any:
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
fail(f"invalid JSON {path}: {exc}")
def resolve_repo_path(relative: str) -> Path:
path = (REPO_ROOT / relative).resolve()
if not path.is_relative_to(REPO_ROOT.resolve()):
fail(f"input escapes repository: {relative}")
return path
def verify_records(
records: list[dict[str, Any]], *, base: Path, expected_names: set[str] | None = None
) -> None:
names = []
for record in records:
if set(record) != {"path", "sha256", "bytes"}:
fail(f"malformed hash record: {record}")
relative = record["path"]
if not isinstance(relative, str) or not relative:
fail("hash record path is missing")
path = (base / relative).resolve()
if not path.is_relative_to(base.resolve()):
fail(f"hash record escapes base: {relative}")
if not path.is_file() or path.is_symlink():
fail(f"hashed file missing or symlinked: {path}")
if path.stat().st_size != record["bytes"]:
fail(f"byte count mismatch: {path}")
if sha256_file(path) != record["sha256"]:
fail(f"SHA-256 mismatch: {path}")
names.append(relative)
if len(names) != len(set(names)):
fail("duplicate hash records")
if expected_names is not None and set(names) != expected_names:
fail(f"artifact set mismatch: {set(names)} != {expected_names}")
def decode_image_url(url: str) -> bytes:
match = re.fullmatch(r"data:image/[A-Za-z0-9.+-]+;base64,([A-Za-z0-9+/=]+)", url)
if not match:
fail("judge request does not contain an exact base64 image data URL")
try:
return base64.b64decode(match.group(1), validate=True)
except ValueError as exc:
fail(f"invalid image base64: {exc}")
def validate_receipts(
retained: dict[str, Any], receipts_doc: dict[str, Any], summary: dict[str, Any]
) -> list[dict[str, Any]]:
if (
receipts_doc.get("schema_version") != "exp8-4-judge-receipts-v1"
or receipts_doc.get("experiment") != "8-4"
):
fail("wrong judge receipt schema or experiment")
if receipts_doc.get("credential_headers_retained") is not False:
fail("judge receipt must state that credential headers were not retained")
receipts = receipts_doc.get("calls")
if not isinstance(receipts, list) or [row.get("image") for row in receipts] != list(
audit.IMAGE_FILES
):
fail("judge calls must cover the eight images exactly in canonical order")
response_ids: list[str] = []
for receipt in receipts:
image = receipt["image"]
if receipt.get("image_source_filename") != audit.IMAGE_FILES[image]:
fail(f"wrong source image filename for {image}")
if receipt.get("image_sha256") != audit.IMAGE_SHA256[image]:
fail(f"wrong image SHA-256 for {image}")
if not isinstance(receipt.get("image_bytes"), int) or receipt["image_bytes"] <= 0:
fail(f"missing image byte count for {image}")
if receipt.get("provider") != "ark" or receipt.get("credential_env") != "ARK_API_KEY":
fail(f"wrong provider metadata for {image}")
if receipt.get("credential_headers_retained") is not False:
fail(f"credential header retention is not false for {image}")
if (
receipt.get("http_status") != 200
or not isinstance(receipt.get("latency_ms"), (int, float))
or receipt["latency_ms"] <= 0
):
fail(f"invalid transport evidence for {image}")
if receipt.get("blind_seed") != audit.BLIND_SEED or receipt.get(
"blind_map"
) != audit.blind_mapping(image):
fail(f"blind mapping mismatch for {image}")
if set(receipt["blind_map"]) != set(audit.LABELS) or set(
receipt["blind_map"].values()
) != set(audit.CONFIGS):
fail(f"blind map is not a bijection for {image}")
request = receipt.get("request")
if not isinstance(request, dict) or request.get("temperature") != 0:
fail(f"malformed deterministic judge request for {image}")
if request.get("response_format") != {"type": "json_object"}:
fail(f"judge request is not fail-closed JSON mode for {image}")
if request.get("model") != summary["judge"]["model"]:
fail(f"judge model mismatch for {image}")
messages = request.get("messages")
if (
not isinstance(messages, list)
or len(messages) != 2
or messages[0].get("role") != "system"
):
fail(f"malformed judge messages for {image}")
user_content = messages[1].get("content")
if not isinstance(user_content, list) or len(user_content) != 2:
fail(f"image-aware user content missing for {image}")
image_part, text_part = user_content
if (
image_part.get("type") != "image_url"
or image_part.get("image_url", {}).get("detail") != "high"
):
fail(f"high-detail image input missing for {image}")
raw_image = decode_image_url(image_part["image_url"].get("url", ""))
if (
hashlib.sha256(raw_image).hexdigest() != audit.IMAGE_SHA256[image]
or len(raw_image) != receipt["image_bytes"]
):
fail(f"request image bytes do not match pinned input for {image}")
if text_part.get("type") != "text" or not isinstance(text_part.get("text"), str):
fail(f"judge text input missing for {image}")
try:
prompt = json.loads(text_part["text"])
except json.JSONDecodeError as exc:
fail(f"judge text is not canonical JSON for {image}: {exc}")
expected_outputs = audit.outputs_for_image(retained, image)
expected_candidates = {
label: expected_outputs[config] for label, config in receipt["blind_map"].items()
}
if prompt.get("image") != image or prompt.get("candidates") != expected_candidates:
fail(f"anonymous candidate text does not match retained outputs for {image}")
prompt_text = json.dumps(prompt, ensure_ascii=False)
if any(config in prompt_text for config in audit.CONFIGS):
fail(f"judge prompt leaks configuration identity for {image}")
response = receipt.get("response")
response_id = receipt.get("response_id")
if (
not isinstance(response, dict)
or response.get("id") != response_id
or not isinstance(response_id, str)
or not response_id
):
fail(f"raw response ID mismatch for {image}")
if (
response.get("usage") != receipt.get("usage")
or not isinstance(receipt.get("usage", {}).get("total_tokens"), int)
or receipt["usage"]["total_tokens"] <= 0
):
fail(f"raw response usage mismatch for {image}")
try:
parsed = audit.extract_json_object(response["choices"][0]["message"]["content"])
audit.validate_judgment(parsed, image)
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
fail(f"invalid raw judgment for {image}: {exc}")
if parsed != receipt.get("judgment"):
fail(f"normalized judgment is not derived from raw response for {image}")
response_ids.append(response_id)
if len(set(response_ids)) != len(audit.IMAGE_FILES):
fail("judge response IDs are not unique")
return receipts
def scan_credentials(run_dir: Path) -> None:
for path in run_dir.iterdir():
if not path.is_file() or path.suffix not in {".json", ".md"}:
continue
text = path.read_text(encoding="utf-8")
for pattern in FORBIDDEN_SECRET_PATTERNS:
if pattern.search(text):
fail(f"possible credential material in {path.name}")
def validate_run(run_dir: Path, *, verify_latest: bool = True) -> dict[str, Any]:
run_dir = run_dir.resolve()
if not run_dir.is_dir():
fail(f"missing run directory: {run_dir}")
if any(path.is_symlink() for path in run_dir.iterdir()):
fail("run directory contains a symlink")
manifest = load_json(run_dir / "manifest.json")
if (
manifest.get("schema_version") != "exp8-4-manifest-v1"
or manifest.get("experiment") != "8-4"
):
fail("wrong manifest schema or experiment")
if (
manifest.get("status") != "passed"
or manifest.get("checkpoint_policy") != "not distributed; not an acceptance artifact"
):
fail("manifest does not declare a passed checkpoint-free report")
if not isinstance(manifest.get("inputs"), list) or not isinstance(
manifest.get("artifacts"), list
):
fail("manifest hash lists are missing")
for record in manifest["inputs"]:
path = resolve_repo_path(record.get("path", ""))
if (
not path.is_file()
or path.is_symlink()
or path.stat().st_size != record.get("bytes")
or sha256_file(path) != record.get("sha256")
):
fail(f"input hash mismatch: {record.get('path')}")
if len({record["path"] for record in manifest["inputs"]}) != len(manifest["inputs"]):
fail("duplicate manifest inputs")
verify_records(manifest["artifacts"], base=run_dir, expected_names=EXPECTED_ARTIFACTS)
retained = load_json(run_dir / "retained_outputs.json")
current_retained = audit.parse_retained_outputs()
if retained != current_retained:
fail("retained outputs do not exactly match the content-hashed book report")
if retained.get("cell_count") != 8 or retained.get("output_count") != 64:
fail("retained output coverage is incomplete")
if retained.get("configs") != list(audit.CONFIGS) or retained.get("images") != list(
audit.IMAGE_FILES
):
fail("retained configuration/image contract changed")
contract = load_json(run_dir / "reproduction_contract.json")
if contract != audit.reproduction_contract():
fail("reproduction contract differs from frozen source/data/model pins")
if contract["checkpoint_policy"]["acceptance_artifact"] is not False:
fail("checkpoint policy was weakened")
summary = load_json(run_dir / "summary.json")
if summary.get("schema_version") != "exp8-4-summary-v1" or summary.get("status") != "passed":
fail("summary is not a passed Experiment 8-4 report")
receipts_doc = load_json(run_dir / "judge_receipts.json")
receipts = validate_receipts(retained, receipts_doc, summary)
recomputed = audit.summarize(retained, receipts, contract)
if summary != recomputed:
fail("summary metrics or acceptance gates do not recompute exactly")
if manifest.get("acceptance") != summary.get("acceptance") or not all(
summary["acceptance"].values()
):
fail("manifest/summary acceptance mismatch")
report = (run_dir / "report.md").read_text(encoding="utf-8")
if (
"Status: **passed**" not in report
or summary["scientific_findings"]["top_configuration"] not in report
):
fail("rendered report does not bind the recomputed result")
scan_credentials(run_dir)
if verify_latest:
latest = load_json(audit.LATEST_PATH)
if latest.get("schema_version") != "exp8-4-latest-v1" or latest.get("experiment") != "8-4":
fail("latest pointer has wrong schema or experiment")
if latest.get("run_id") != manifest.get("run_id") or latest.get("status") != "passed":
fail("latest pointer does not identify this passed run")
if latest.get("manifest_sha256") != sha256_file(run_dir / "manifest.json"):
fail("latest pointer manifest SHA-256 mismatch")
expected_run_dir = EXPERIMENT_DIR / latest.get("run_dir", "")
if expected_run_dir.resolve() != run_dir:
fail("latest pointer resolves to another run")
return {
"experiment": "8-4",
"status": "passed",
"run_id": manifest["run_id"],
"cells": retained["cell_count"],
"outputs": retained["output_count"],
"images": len(audit.IMAGE_FILES),
"judge_receipts": len(receipts),
"artifacts": len(manifest["artifacts"]),
"inputs": len(manifest["inputs"]),
"manifest_sha256": sha256_file(run_dir / "manifest.json"),
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--run-dir", type=Path, help="Run directory; defaults to validation/latest_vlm.json"
)
parser.add_argument(
"--no-latest",
action="store_true",
help="Skip latest-pointer binding (for deliberate tamper tests only)",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
if args.run_dir is None:
latest = load_json(audit.LATEST_PATH)
run_dir = EXPERIMENT_DIR / latest.get("run_dir", "")
else:
run_dir = args.run_dir
try:
result = validate_run(run_dir, verify_latest=not args.no_latest)
except EvidenceError as exc:
print(
json.dumps(
{"experiment": "8-4", "status": "failed", "error": str(exc)},
indent=2,
sort_keys=True,
)
)
return 1
print(json.dumps(result, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())