liqiang
b119135836
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
761 lines
34 KiB
Python
761 lines
34 KiB
Python
#!/usr/bin/env python3
|
|
"""Build checkpoint-free retained training evidence for Experiment 8-3.
|
|
|
|
The book already contains the author's historical six-cell evaluation report:
|
|
original versus QK-Norm + Muon at pretrain, SFT, and DPO. This program does
|
|
not pretend to rerun the GPU training job. It extracts every saved generation,
|
|
submits a preregistered stage-balanced subset to an arm-blind external judge,
|
|
and binds the raw report, judge receipts, immutable future-reproduction source
|
|
and dataset revisions, environment lock, findings, and limitations into a
|
|
content-hashed evidence package. Checkpoints are intentionally not published
|
|
and are not an acceptance artifact for book training experiments.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import random
|
|
import re
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
EXPERIMENT_DIR = HERE.parent
|
|
REPO_ROOT = EXPERIMENT_DIR.parents[1]
|
|
REPORT_PATH = EXPERIMENT_DIR / "README.md"
|
|
RUNS_DIR = HERE / "runs"
|
|
LATEST_PATH = HERE / "latest.json"
|
|
|
|
DEFAULT_RUN_ID = "exp8-3-training-report-20260731-v1"
|
|
DEFAULT_ENDPOINT = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
|
|
DEFAULT_MODEL = "doubao-seed-1-6-250615"
|
|
BLIND_SEED = 730731
|
|
|
|
SOURCE_REVISION = "8bdc5d97d5845a8c1ac2ed56a5b8b4c0d0fb0795"
|
|
DATASET_REVISION = "84983ed4dec7836d240577760c1d6be5d4cabcf9"
|
|
SOURCE_FILES = {
|
|
"model/model_minimind.py": "2d33988711c704be6a22c4c61489b23106a2340a7cb8b97ebe3e40f30819cbb0",
|
|
"model/tokenizer.json": "e489029175fb3f94b8211a120a72a2ee41a664db65b828d077c7bde989c845a9",
|
|
"model/tokenizer_config.json": "190cc4738bac3b6f6b563376019c581b320fdb0260a03b9d5ab806296c8c6bb8",
|
|
"dataset/lm_dataset.py": "213726b1781289548784220b2b2db48fe97d84f3b67ac0cd70186cbbcb7b5d2c",
|
|
"trainer/muon.py": "00c2c6a225edeb55433df0724c3c74f6ff98ac4b2cc73c4aafcff686824f6267",
|
|
"trainer/train_pretrain.py": "ddd122645a9f1043bc8dac69a81ac51d2df95df8745d25faed7963d38fedc328",
|
|
"trainer/train_pretrain_muon.py": "fc83d07754ec3a8c156b6b8bfc0fd4326edecb72efabc5e08ae4ff5e3a7029bc",
|
|
"trainer/train_full_sft.py": "a57422f1df80bf2867f31f3b4a646a92ac7f66729e98a3b32cd1ec4d6780cb8b",
|
|
"trainer/train_full_sft_muon.py": "acd0b7db5b1d8b25d3c3103f92d68a7d381f9322a1b33bbec34be3d005930bad",
|
|
"trainer/train_dpo.py": "97f2c31cc8bc21a777e2efcb5e2fa35a49e4e9e3698db120148f8a0b2f678449",
|
|
"eval_model.py": "43930a4b55048a4a3ffa17eb78ae67d59582d639aa9365f0bbf41ba149128af8",
|
|
"requirements.txt": "23f4cea09281765eec7cf03e28231425638e8e42580f418781fed166b75af968",
|
|
}
|
|
DATASET_FILES = {
|
|
"pretrain_hq.jsonl": {
|
|
"lfs_sha256": "9801b0d2210c61c2e4bc130f6dc4b3c870698a88d04af8f103c23dd5f0ce2440",
|
|
"bytes": 1_669_750_047,
|
|
},
|
|
"sft_512.jsonl": {
|
|
"lfs_sha256": "053b7d09574e48a86232e929211434ff9e5016c6ed13312e63687dd52edcbebf",
|
|
"bytes": 7_531_517_862,
|
|
},
|
|
"dpo.jsonl": {
|
|
"lfs_sha256": "ee934a8a455ccc99d1334d63e1254dd1d64f497fd067cfcbb71e3043f5b46768",
|
|
"bytes": 53_653_322,
|
|
},
|
|
}
|
|
|
|
ARMS = ("original", "qk_norm_muon")
|
|
STAGES = ("pretrain", "sft", "dpo")
|
|
EXPECTED_COUNTS = {
|
|
("original", "pretrain"): 7,
|
|
("original", "sft"): 8,
|
|
("original", "dpo"): 9,
|
|
("qk_norm_muon", "pretrain"): 7,
|
|
("qk_norm_muon", "sft"): 9,
|
|
("qk_norm_muon", "dpo"): 9,
|
|
}
|
|
SELECTED_CASES = (
|
|
{"case_id": 1, "stage": "pretrain", "keyword": "highest mountain", "task": "Continue the prompt by identifying the highest mountain in the world accurately."},
|
|
{"case_id": 2, "stage": "pretrain", "keyword": "carbon dioxide", "task": "Continue the prompt with an accurate statement about carbon dioxide in air."},
|
|
{"case_id": 3, "stage": "sft", "keyword": "speed of light", "task": "Explain the physical concept of the speed of light in detail."},
|
|
{"case_id": 4, "stage": "sft", "keyword": "how to understand chatgpt", "task": "Explain what ChatGPT is and how it works."},
|
|
{"case_id": 5, "stage": "sft", "keyword": "history of the united states", "task": "Introduce the history of the United States."},
|
|
{"case_id": 6, "stage": "dpo", "keyword": "speed of light", "task": "Explain the physical concept of the speed of light in detail."},
|
|
{"case_id": 7, "stage": "dpo", "keyword": "how to understand chatgpt", "task": "Explain what ChatGPT is and how it works."},
|
|
{"case_id": 8, "stage": "dpo", "keyword": "history of the united states", "task": "Introduce the history of the United States."},
|
|
)
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def canonical_json_bytes(value: Any) -> bytes:
|
|
return (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode()
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.write_bytes(canonical_json_bytes(value))
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def _terminal_transcript(section: str) -> tuple[str, str]:
|
|
"""Return the terminal header and prompt/output transcript from one cell.
|
|
|
|
The historical Markdown has one stray closing fence immediately before the
|
|
improved-SFT Lu Xun answer. Treat fences as presentation markup rather
|
|
than semantic delimiters so that the retained answer is not silently lost.
|
|
"""
|
|
|
|
prompt_position = section.find("👶:")
|
|
if prompt_position < 0:
|
|
raise ValueError("model section has no user prompt marker")
|
|
header = section[:prompt_position].replace("```", "").strip()
|
|
transcript = section[prompt_position:]
|
|
analysis = re.search(r"(?m)^\*\*[^\n]*Analysis[^\n]*\*\*:?\s*$", transcript)
|
|
if analysis:
|
|
transcript = transcript[: analysis.start()]
|
|
transcript = re.sub(r"(?m)^```\s*$", "", transcript).strip()
|
|
return header, transcript
|
|
|
|
|
|
def _parse_pairs(transcript: str) -> list[dict[str, str]]:
|
|
pattern = re.compile(
|
|
r"^👶:\s*(.*?)\n🤖️:\s*(.*?)(?=\n(?:\s*\n)*👶:|\Z)",
|
|
flags=re.DOTALL | re.MULTILINE,
|
|
)
|
|
pairs = []
|
|
for match in pattern.finditer(transcript):
|
|
prompt = match.group(1).strip()
|
|
output = match.group(2).strip()
|
|
if not prompt or not output:
|
|
raise ValueError("empty prompt or output in historical transcript")
|
|
pairs.append({"prompt": prompt, "output": output})
|
|
return pairs
|
|
|
|
|
|
def parse_retained_outputs(report_path: Path = REPORT_PATH) -> dict[str, Any]:
|
|
"""Extract all six historical LLM evaluation cells from the bilingual report."""
|
|
|
|
text = report_path.read_text(encoding="utf-8")
|
|
start = text.index("## Language Model Training Results Analysis")
|
|
end = text.index("# Analysis of Vision-Language Model Training Results", start)
|
|
llm_text = text[start:end]
|
|
arm_markers = {
|
|
"original": "## Without Muon Optimizer (Original Architecture)",
|
|
"qk_norm_muon": "## With Muon Optimizer and QK Norm (Improved Architecture)",
|
|
}
|
|
cells: list[dict[str, Any]] = []
|
|
for arm_index, arm in enumerate(ARMS):
|
|
arm_start = llm_text.index(arm_markers[arm])
|
|
arm_end = (
|
|
llm_text.index(arm_markers[ARMS[arm_index + 1]], arm_start)
|
|
if arm_index + 1 < len(ARMS)
|
|
else len(llm_text)
|
|
)
|
|
arm_text = llm_text[arm_start:arm_end]
|
|
for stage_index, stage in enumerate(STAGES):
|
|
heading = {"pretrain": "### Pretrain Model", "sft": "### SFT Model", "dpo": "### DPO Model"}[stage]
|
|
cell_start = arm_text.index(heading)
|
|
next_positions = [
|
|
arm_text.find(next_heading, cell_start + len(heading))
|
|
for next_heading in ("### Pretrain Model", "### SFT Model", "### DPO Model")
|
|
]
|
|
next_positions = [position for position in next_positions if position >= 0]
|
|
cell_end = min(next_positions) if next_positions else len(arm_text)
|
|
header, transcript = _terminal_transcript(arm_text[cell_start:cell_end])
|
|
pairs = _parse_pairs(transcript)
|
|
expected = EXPECTED_COUNTS[(arm, stage)]
|
|
if len(pairs) != expected:
|
|
raise ValueError(f"{arm}/{stage}: expected {expected} pairs, found {len(pairs)}")
|
|
cells.append(
|
|
{
|
|
"arm": arm,
|
|
"stage": stage,
|
|
"terminal_header": header,
|
|
"pair_count": len(pairs),
|
|
"pairs": pairs,
|
|
}
|
|
)
|
|
|
|
return {
|
|
"schema_version": "exp8-3-retained-outputs-v1",
|
|
"experiment": "8-3",
|
|
"source_report": str(report_path.relative_to(REPO_ROOT)),
|
|
"source_report_sha256": sha256_file(report_path),
|
|
"arms": list(ARMS),
|
|
"stages": list(STAGES),
|
|
"cell_count": len(cells),
|
|
"output_count": sum(cell["pair_count"] for cell in cells),
|
|
"cells": cells,
|
|
}
|
|
|
|
|
|
def _find_pair(retained: dict[str, Any], arm: str, stage: str, keyword: str) -> dict[str, str]:
|
|
cell = next(cell for cell in retained["cells"] if cell["arm"] == arm and cell["stage"] == stage)
|
|
matches = [pair for pair in cell["pairs"] if keyword in pair["prompt"].lower()]
|
|
if len(matches) != 1:
|
|
raise ValueError(f"{arm}/{stage}/{keyword}: expected one prompt, found {len(matches)}")
|
|
return matches[0]
|
|
|
|
|
|
def selected_comparisons(retained: dict[str, Any]) -> list[dict[str, Any]]:
|
|
comparisons = []
|
|
for case in SELECTED_CASES:
|
|
rows = {
|
|
arm: _find_pair(retained, arm, case["stage"], case["keyword"])
|
|
for arm in ARMS
|
|
}
|
|
comparisons.append({**case, "arms": rows})
|
|
return comparisons
|
|
|
|
|
|
def blind_mapping(case_id: int) -> dict[str, str]:
|
|
arms = list(ARMS)
|
|
random.Random(BLIND_SEED + case_id).shuffle(arms)
|
|
return dict(zip(("A", "B"), arms, strict=True))
|
|
|
|
|
|
def judge_payload(comparison: dict[str, Any], mapping: dict[str, str], model: str) -> dict[str, Any]:
|
|
candidates = {
|
|
label: {
|
|
"historical_prompt": comparison["arms"][arm]["prompt"],
|
|
"historical_output": comparison["arms"][arm]["output"],
|
|
}
|
|
for label, arm in mapping.items()
|
|
}
|
|
required = {
|
|
"case_id": comparison["case_id"],
|
|
"candidates": {
|
|
label: {
|
|
"language_fluency": "number 0-5",
|
|
"instruction_following": "number 0-5",
|
|
"factuality": "number 0-5",
|
|
"factual_errors": ["specific material errors; empty only if none"],
|
|
"rationale": "brief evidence-based explanation",
|
|
}
|
|
for label in ("A", "B")
|
|
},
|
|
"winner": "A, B, or tie",
|
|
}
|
|
return {
|
|
"model": model,
|
|
"temperature": 0,
|
|
"response_format": {"type": "json_object"},
|
|
"messages": [
|
|
{
|
|
"role": "system",
|
|
"content": (
|
|
"You are an independent evaluator of two anonymous small-language-model outputs. "
|
|
"Do not infer model identity, architecture, optimizer, or training stage. Score only "
|
|
"the supplied text. Penalize hallucinations, unsafe medical specificity, repetition, "
|
|
"and non-answers. Return one JSON object with exactly the requested fields."
|
|
),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": json.dumps(
|
|
{
|
|
"case_id": comparison["case_id"],
|
|
"task": comparison["task"],
|
|
"rubric": {
|
|
"language_fluency": "0 unreadable; 3 understandable with defects; 5 coherent and natural",
|
|
"instruction_following": "0 non-answer; 3 partial; 5 directly and fully answers",
|
|
"factuality": "0 dominated by falsehoods; 3 mixed/minor errors; 5 no material error",
|
|
},
|
|
"candidates": candidates,
|
|
"required_json_shape": required,
|
|
},
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
),
|
|
},
|
|
],
|
|
}
|
|
|
|
|
|
def extract_json_object(content: str) -> dict[str, Any]:
|
|
stripped = content.strip()
|
|
if stripped.startswith("```"):
|
|
stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
|
|
stripped = re.sub(r"\s*```$", "", stripped)
|
|
parsed = json.loads(stripped)
|
|
if not isinstance(parsed, dict):
|
|
raise ValueError("judge content must decode to an object")
|
|
return parsed
|
|
|
|
|
|
def validate_judgment(judgment: dict[str, Any], case_id: int) -> None:
|
|
if str(judgment.get("case_id")) != str(case_id):
|
|
raise ValueError(
|
|
f"judge returned the wrong case_id: expected {case_id}, got {judgment.get('case_id')!r}"
|
|
)
|
|
candidates = judgment.get("candidates")
|
|
if not isinstance(candidates, dict) or set(candidates) != {"A", "B"}:
|
|
raise ValueError("judge must score A and B exactly")
|
|
for label in ("A", "B"):
|
|
row = candidates[label]
|
|
if not isinstance(row, dict):
|
|
raise ValueError(f"candidate {label} score must be an object")
|
|
for metric in ("language_fluency", "instruction_following", "factuality"):
|
|
score = row.get(metric)
|
|
if not isinstance(score, (int, float)) or isinstance(score, bool) or not 0 <= score <= 5:
|
|
raise ValueError(f"candidate {label} has invalid {metric}: {score!r}")
|
|
if not isinstance(row.get("factual_errors"), list):
|
|
raise ValueError(f"candidate {label} factual_errors must be a list")
|
|
if not isinstance(row.get("rationale"), str) or not row["rationale"].strip():
|
|
raise ValueError(f"candidate {label} rationale is missing")
|
|
if judgment.get("winner") not in {"A", "B", "tie"}:
|
|
raise ValueError("judge winner must be A, B, or tie")
|
|
|
|
|
|
def call_judge(
|
|
comparison: dict[str, Any], *, endpoint: str, model: str, api_key: str, timeout: float
|
|
) -> dict[str, Any]:
|
|
mapping = blind_mapping(comparison["case_id"])
|
|
payload = judge_payload(comparison, mapping, model)
|
|
request = urllib.request.Request(
|
|
endpoint,
|
|
data=json.dumps(payload, ensure_ascii=False).encode(),
|
|
method="POST",
|
|
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
|
)
|
|
started = time.perf_counter()
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
raw_body = response.read()
|
|
http_status = response.status
|
|
except urllib.error.HTTPError as exc:
|
|
body = exc.read().decode(errors="replace")
|
|
raise RuntimeError(f"judge HTTP {exc.code}: {body[:500]}") from exc
|
|
latency_ms = round((time.perf_counter() - started) * 1000, 3)
|
|
raw_response = json.loads(raw_body)
|
|
content = raw_response["choices"][0]["message"]["content"]
|
|
judgment = extract_json_object(content)
|
|
validate_judgment(judgment, comparison["case_id"])
|
|
response_id = raw_response.get("id")
|
|
usage = raw_response.get("usage")
|
|
if not isinstance(response_id, str) or not response_id:
|
|
raise ValueError("judge response has no response ID")
|
|
if not isinstance(usage, dict) or not isinstance(usage.get("total_tokens"), int):
|
|
raise ValueError("judge response has no complete usage object")
|
|
return {
|
|
"case_id": comparison["case_id"],
|
|
"stage": comparison["stage"],
|
|
"keyword": comparison["keyword"],
|
|
"task": comparison["task"],
|
|
"provider": "ark",
|
|
"endpoint": endpoint,
|
|
"credential_env": "ARK_API_KEY",
|
|
"credential_headers_retained": False,
|
|
"blind_seed": BLIND_SEED,
|
|
"blind_map": mapping,
|
|
"request": payload,
|
|
"http_status": http_status,
|
|
"response": raw_response,
|
|
"response_id": response_id,
|
|
"usage": usage,
|
|
"latency_ms": latency_ms,
|
|
"judgment": judgment,
|
|
}
|
|
|
|
|
|
def reproduction_contract() -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "exp8-3-reproduction-contract-v1",
|
|
"experiment": "8-3",
|
|
"historical_evidence_boundary": {
|
|
"historical_training_executed": True,
|
|
"six_historical_evaluation_transcripts_retained": True,
|
|
"historical_source_revision_retained": False,
|
|
"historical_dataset_hashes_retained": False,
|
|
"historical_checkpoint_hashes_retained": False,
|
|
"historical_stepwise_training_logs_retained": False,
|
|
"claim": (
|
|
"The author's retained report supports that original and QK-Norm+Muon 104.03M models "
|
|
"were evaluated after pretrain, SFT, and DPO. It does not establish byte identity of the "
|
|
"historical checkpoints, datasets, source checkout, or every loss point."
|
|
),
|
|
},
|
|
"future_reproduction": {
|
|
"source": {
|
|
"repository": "bojieli/minimind",
|
|
"revision": SOURCE_REVISION,
|
|
"selected_at": "2026-07-31",
|
|
"not_claimed_as_historical_revision": True,
|
|
"file_sha256": SOURCE_FILES,
|
|
},
|
|
"dataset": {
|
|
"repository": "jingyaogong/minimind_dataset",
|
|
"revision": DATASET_REVISION,
|
|
"selected_at": "2026-07-31",
|
|
"not_claimed_as_historical_revision": True,
|
|
"files": DATASET_FILES,
|
|
},
|
|
"commands": {
|
|
"original_pretrain": "torchrun --nproc_per_node=8 trainer/train_pretrain.py --epochs 10 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/pretrain_hq.jsonl --use_wandb",
|
|
"improved_pretrain": "torchrun --nproc_per_node=8 trainer/train_pretrain_muon.py --epochs 10 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/pretrain_hq.jsonl --use_wandb",
|
|
"original_sft": "torchrun --nproc_per_node=8 trainer/train_full_sft.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/sft_512.jsonl --use_wandb",
|
|
"improved_sft": "torchrun --nproc_per_node=8 trainer/train_full_sft_muon.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --data_path dataset/sft_512.jsonl --use_wandb",
|
|
"original_dpo": "torchrun --nproc_per_node=8 trainer/train_dpo.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --init_from out/full_sft_768.pth --data_path dataset/dpo.jsonl --use_wandb",
|
|
"improved_dpo": "torchrun --nproc_per_node=8 trainer/train_dpo.py --epochs 2 --hidden_size 768 --num_hidden_layers 16 --init_from out/full_sft_muon_768.pth --data_path dataset/dpo.jsonl --use_wandb",
|
|
},
|
|
"environment": {
|
|
"book_pyproject": "pyproject.toml",
|
|
"book_lock": "uv.lock",
|
|
"install": "uv sync --locked --python 3.12 --extra ch7 --extra dev",
|
|
"boundary": (
|
|
"The book lock freezes a future software environment. The pinned upstream requirements "
|
|
"file itself is unversioned, and the future GPU/CUDA stack has not been exercised here."
|
|
),
|
|
},
|
|
},
|
|
"model_and_training": {
|
|
"reported_parameter_count_millions": 104.03,
|
|
"architecture": {"hidden_size": 768, "layers": 16, "sequence_length": 512},
|
|
"stages": list(STAGES),
|
|
"arms": list(ARMS),
|
|
"source_verified_mechanisms": {
|
|
"qk_norm_before_rope": True,
|
|
"muon_for_two_dimensional_non_embedding_weights": True,
|
|
"adamw_for_embeddings_norms_and_lm_head": True,
|
|
"dpo_uses_adamw_from_arm_specific_sft_checkpoint": True,
|
|
},
|
|
"reported_scalars_without_stepwise_logs": {
|
|
"steps_to_loss_3_original": 36,
|
|
"steps_to_loss_3_qk_norm_muon": 12,
|
|
"final_loss_original": 2.0,
|
|
"final_loss_qk_norm_muon": 1.7,
|
|
"eight_rtx_4090_pretrain_hours": 6,
|
|
"eight_rtx_4090_sft_hours": 8,
|
|
},
|
|
},
|
|
"checkpoint_policy": {
|
|
"distributed_with_book": False,
|
|
"acceptance_artifact": False,
|
|
"required_artifact": "reproducible evidence-backed training report",
|
|
"reason": "Training checkpoints are intentionally not distributed to readers.",
|
|
},
|
|
}
|
|
|
|
|
|
def mean(values: list[float]) -> float:
|
|
return round(sum(values) / len(values), 4)
|
|
|
|
|
|
def summarize(retained: dict[str, Any], receipts: list[dict[str, Any]], contract: dict[str, Any]) -> dict[str, Any]:
|
|
metrics = ("language_fluency", "instruction_following", "factuality")
|
|
rows: dict[int, dict[str, dict[str, Any]]] = {}
|
|
wins = {"original": 0, "qk_norm_muon": 0, "tie": 0}
|
|
for receipt in receipts:
|
|
reverse = receipt["blind_map"]
|
|
rows[receipt["case_id"]] = {
|
|
reverse[label]: score for label, score in receipt["judgment"]["candidates"].items()
|
|
}
|
|
winner = receipt["judgment"]["winner"]
|
|
wins["tie" if winner == "tie" else reverse[winner]] += 1
|
|
|
|
arm_averages = {}
|
|
stage_averages = {}
|
|
for arm in ARMS:
|
|
arm_scores = [rows[case["case_id"]][arm] for case in SELECTED_CASES]
|
|
arm_averages[arm] = {
|
|
metric: mean([float(row[metric]) for row in arm_scores]) for metric in metrics
|
|
}
|
|
arm_averages[arm]["overall"] = mean(
|
|
[float(row[metric]) for row in arm_scores for metric in metrics]
|
|
)
|
|
stage_averages[arm] = {}
|
|
for stage in STAGES:
|
|
stage_scores = [
|
|
rows[case["case_id"]][arm] for case in SELECTED_CASES if case["stage"] == stage
|
|
]
|
|
stage_averages[arm][stage] = {
|
|
metric: mean([float(row[metric]) for row in stage_scores]) for metric in metrics
|
|
}
|
|
stage_averages[arm][stage]["overall"] = mean(
|
|
[float(row[metric]) for row in stage_scores for metric in metrics]
|
|
)
|
|
|
|
findings = {
|
|
"blind_judge_overall_delta_qk_norm_muon_minus_original": round(
|
|
arm_averages["qk_norm_muon"]["overall"] - arm_averages["original"]["overall"], 4
|
|
),
|
|
"blind_judge_prefers_qk_norm_muon_overall": (
|
|
arm_averages["qk_norm_muon"]["overall"] > arm_averages["original"]["overall"]
|
|
),
|
|
"wins": wins,
|
|
"reported_loss_comparison_retained_but_not_independently_recomputed": True,
|
|
}
|
|
acceptance = {
|
|
"raw_historical_report_hashed": bool(retained["source_report_sha256"]),
|
|
"all_six_arm_stage_cells_retained": retained["cell_count"] == 6,
|
|
"all_expected_outputs_retained": retained["output_count"] == sum(EXPECTED_COUNTS.values()),
|
|
"pretrain_sft_and_dpo_compared": set(retained["stages"]) == set(STAGES),
|
|
"original_and_qk_norm_muon_compared": set(retained["arms"]) == set(ARMS),
|
|
"eight_stage_balanced_blind_judgments": len(receipts) == len(SELECTED_CASES),
|
|
"raw_judge_requests_responses_ids_usage_latency_retained": all(
|
|
receipt["response_id"]
|
|
and receipt["usage"].get("total_tokens", 0) > 0
|
|
and receipt["latency_ms"] > 0
|
|
for receipt in receipts
|
|
),
|
|
"immutable_source_revision_and_file_hashes_frozen": bool(SOURCE_REVISION and SOURCE_FILES),
|
|
"immutable_dataset_revision_lfs_hashes_and_sizes_frozen": bool(DATASET_REVISION and DATASET_FILES),
|
|
"future_reproduction_commands_declared": len(contract["future_reproduction"]["commands"]) == 6,
|
|
"historical_provenance_limitations_explicit": (
|
|
contract["historical_evidence_boundary"]["historical_checkpoint_hashes_retained"] is False
|
|
and contract["historical_evidence_boundary"]["historical_stepwise_training_logs_retained"] is False
|
|
),
|
|
"reported_loss_claims_qualified": findings[
|
|
"reported_loss_comparison_retained_but_not_independently_recomputed"
|
|
],
|
|
"checkpoints_not_an_acceptance_artifact": (
|
|
contract["checkpoint_policy"]["acceptance_artifact"] is False
|
|
),
|
|
}
|
|
passed = all(acceptance.values())
|
|
return {
|
|
"schema_version": "exp8-3-summary-v1",
|
|
"experiment": "8-3",
|
|
"status": "passed" if passed else "failed",
|
|
"judge": {
|
|
"provider": "ark",
|
|
"model": receipts[0]["request"]["model"],
|
|
"calls": len(receipts),
|
|
"response_ids": [receipt["response_id"] for receipt in receipts],
|
|
"total_tokens": sum(receipt["usage"]["total_tokens"] for receipt in receipts),
|
|
"total_latency_ms": round(sum(receipt["latency_ms"] for receipt in receipts), 3),
|
|
"blind_seed": BLIND_SEED,
|
|
},
|
|
"retained": {
|
|
"cells": retained["cell_count"],
|
|
"outputs": retained["output_count"],
|
|
"selected_comparisons": len(receipts),
|
|
},
|
|
"arm_averages": arm_averages,
|
|
"stage_averages": stage_averages,
|
|
"per_case_arm_scores": rows,
|
|
"scientific_findings": findings,
|
|
"acceptance": {**acceptance, "passed": passed},
|
|
"limitations": [
|
|
"Historical checkpoints are intentionally not distributed and were not recreated in this audit.",
|
|
"The historical source revision, dataset byte identities, RNG state, and stepwise loss logs were not retained.",
|
|
"Frozen source/data revisions and the book lock define a future reproduction contract, not historical provenance.",
|
|
"The independent judge covers eight preregistered comparisons; all other retained outputs remain available for inspection.",
|
|
"The historical outputs are English translations in a bilingual report, so translation may affect the judge scores.",
|
|
],
|
|
}
|
|
|
|
|
|
def render_report(summary: dict[str, Any]) -> str:
|
|
averages = summary["arm_averages"]
|
|
findings = summary["scientific_findings"]
|
|
return "\n".join(
|
|
[
|
|
"# Experiment 8-3 retained-training-report audit",
|
|
"",
|
|
"## Result",
|
|
"",
|
|
f"Status: **{summary['status']}**. The historical report retains "
|
|
f"{summary['retained']['outputs']} outputs across the original and QK-Norm + Muon arms "
|
|
"after pretrain, SFT, and DPO. Eight preregistered arm-blind comparisons were judged "
|
|
"from raw retained text by an independent ARK model.",
|
|
"",
|
|
"| Arm | Fluency | Instruction | Factuality | Overall |",
|
|
"| --- | ---: | ---: | ---: | ---: |",
|
|
f"| Original | {averages['original']['language_fluency']:.4f} | "
|
|
f"{averages['original']['instruction_following']:.4f} | "
|
|
f"{averages['original']['factuality']:.4f} | {averages['original']['overall']:.4f} |",
|
|
f"| QK-Norm + Muon | {averages['qk_norm_muon']['language_fluency']:.4f} | "
|
|
f"{averages['qk_norm_muon']['instruction_following']:.4f} | "
|
|
f"{averages['qk_norm_muon']['factuality']:.4f} | {averages['qk_norm_muon']['overall']:.4f} |",
|
|
"",
|
|
f"Observed blind-judge overall delta: **{findings['blind_judge_overall_delta_qk_norm_muon_minus_original']:+.4f}**. "
|
|
f"Pairwise decisions: {findings['wins']}.",
|
|
"",
|
|
"The report's loss claims (3.0 reached at 36 versus 12 reported steps; final loss 2.0 "
|
|
"versus 1.7) are retained as author-reported observations, not independently recomputed "
|
|
"measurements, because the historical stepwise logs were not preserved.",
|
|
"",
|
|
"## Provenance and reproduction boundary",
|
|
"",
|
|
"`reproduction_contract.json` freezes the MiniMind source revision, hashes the relevant "
|
|
"source files, freezes a dataset revision with the three Git-LFS object hashes and sizes, "
|
|
"and records all six future reproduction commands. These pins were selected for future "
|
|
"reproduction and are not represented as the exact historical checkout.",
|
|
"",
|
|
"Training checkpoints remain local by book policy and are not an acceptance artifact. "
|
|
"The accepted artifact is this content-hashed training report, its raw retained outputs, "
|
|
"raw independent-judge receipts, and explicit limitations.",
|
|
"",
|
|
]
|
|
)
|
|
|
|
|
|
def input_record(path: Path) -> dict[str, Any]:
|
|
return {
|
|
"path": str(path.relative_to(REPO_ROOT)),
|
|
"sha256": sha256_file(path),
|
|
"bytes": path.stat().st_size,
|
|
}
|
|
|
|
|
|
def artifact_record(path: Path, run_dir: Path) -> dict[str, Any]:
|
|
return {
|
|
"path": str(path.relative_to(run_dir)),
|
|
"sha256": sha256_file(path),
|
|
"bytes": path.stat().st_size,
|
|
}
|
|
|
|
|
|
def build_manifest(run_id: str, run_dir: Path, summary: dict[str, Any]) -> dict[str, Any]:
|
|
inputs = [
|
|
input_record(REPORT_PATH),
|
|
input_record(REPO_ROOT / "pyproject.toml"),
|
|
input_record(REPO_ROOT / "uv.lock"),
|
|
input_record(HERE / "run_training_report_audit.py"),
|
|
input_record(HERE / "validate_evidence.py"),
|
|
]
|
|
artifact_paths = [
|
|
run_dir / "retained_outputs.json",
|
|
run_dir / "reproduction_contract.json",
|
|
run_dir / "judge_receipts.json",
|
|
run_dir / "summary.json",
|
|
run_dir / "report.md",
|
|
]
|
|
return {
|
|
"schema_version": "exp8-3-manifest-v1",
|
|
"experiment": "8-3",
|
|
"run_id": run_id,
|
|
"created_at": utc_now(),
|
|
"status": summary["status"],
|
|
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
|
|
"inputs": inputs,
|
|
"artifacts": [artifact_record(path, run_dir) for path in artifact_paths],
|
|
"acceptance": summary["acceptance"],
|
|
"checkpoint_policy": "not distributed; not an acceptance artifact",
|
|
}
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--run-id", default=DEFAULT_RUN_ID)
|
|
parser.add_argument("--endpoint", default=os.getenv("ARK_BASE_URL", DEFAULT_ENDPOINT))
|
|
parser.add_argument("--model", default=os.getenv("ARK_MODEL", DEFAULT_MODEL))
|
|
parser.add_argument("--api-key-env", default="ARK_API_KEY")
|
|
parser.add_argument("--timeout", type=float, default=180.0)
|
|
parser.add_argument("--concurrency", type=int, default=8)
|
|
parser.add_argument(
|
|
"--refresh-manifest",
|
|
action="store_true",
|
|
help="Rehash an existing run after source-only corrections; makes no provider call.",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
if not re.fullmatch(r"[A-Za-z0-9._-]+", args.run_id):
|
|
raise SystemExit("run ID may contain only letters, digits, dot, underscore, and hyphen")
|
|
run_dir = RUNS_DIR / args.run_id
|
|
if args.refresh_manifest:
|
|
if not run_dir.is_dir():
|
|
raise SystemExit(f"cannot refresh missing run: {run_dir}")
|
|
stored_retained = json.loads(
|
|
(run_dir / "retained_outputs.json").read_text(encoding="utf-8")
|
|
)
|
|
current_retained = parse_retained_outputs()
|
|
# Documentation around the raw transcripts may change, but a refresh
|
|
# must never silently replace the generations that were judged.
|
|
if stored_retained.get("cells") != current_retained.get("cells"):
|
|
raise SystemExit(
|
|
"refusing manifest refresh because retained historical outputs changed"
|
|
)
|
|
stored_retained["source_report_sha256"] = current_retained[
|
|
"source_report_sha256"
|
|
]
|
|
write_json(run_dir / "retained_outputs.json", stored_retained)
|
|
summary = json.loads((run_dir / "summary.json").read_text(encoding="utf-8"))
|
|
write_json(run_dir / "manifest.json", build_manifest(args.run_id, run_dir, summary))
|
|
latest = {
|
|
"schema_version": "exp8-3-latest-v1",
|
|
"experiment": "8-3",
|
|
"run_id": args.run_id,
|
|
"status": summary["status"],
|
|
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
|
|
"manifest_sha256": sha256_file(run_dir / "manifest.json"),
|
|
}
|
|
write_json(LATEST_PATH, latest)
|
|
print(json.dumps(latest, indent=2, sort_keys=True))
|
|
return 0
|
|
|
|
if run_dir.exists():
|
|
raise SystemExit(f"refusing to overwrite existing run: {run_dir}")
|
|
api_key = os.getenv(args.api_key_env)
|
|
if not api_key:
|
|
raise SystemExit(f"missing required credential environment variable: {args.api_key_env}")
|
|
|
|
retained = parse_retained_outputs()
|
|
comparisons = selected_comparisons(retained)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
futures = {
|
|
comparison["case_id"]: pool.submit(
|
|
call_judge,
|
|
comparison,
|
|
endpoint=args.endpoint,
|
|
model=args.model,
|
|
api_key=api_key,
|
|
timeout=args.timeout,
|
|
)
|
|
for comparison in comparisons
|
|
}
|
|
receipts = [futures[case_id].result() for case_id in sorted(futures)]
|
|
|
|
contract = reproduction_contract()
|
|
summary = summarize(retained, receipts, contract)
|
|
run_dir.mkdir(parents=True)
|
|
write_json(run_dir / "retained_outputs.json", retained)
|
|
write_json(
|
|
run_dir / "judge_receipts.json",
|
|
{
|
|
"schema_version": "exp8-3-judge-receipts-v1",
|
|
"experiment": "8-3",
|
|
"credential_headers_retained": False,
|
|
"calls": receipts,
|
|
},
|
|
)
|
|
write_json(run_dir / "reproduction_contract.json", contract)
|
|
write_json(run_dir / "summary.json", summary)
|
|
(run_dir / "report.md").write_text(render_report(summary), encoding="utf-8")
|
|
write_json(run_dir / "manifest.json", build_manifest(args.run_id, run_dir, summary))
|
|
latest = {
|
|
"schema_version": "exp8-3-latest-v1",
|
|
"experiment": "8-3",
|
|
"run_id": args.run_id,
|
|
"status": summary["status"],
|
|
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
|
|
"manifest_sha256": sha256_file(run_dir / "manifest.json"),
|
|
}
|
|
write_json(LATEST_PATH, latest)
|
|
print(json.dumps(latest, indent=2, sort_keys=True))
|
|
return 0 if summary["status"] == "passed" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|