liqiang
b119135836
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
314 lines
14 KiB
Python
314 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Fail-closed validator for Experiment 8-3 retained training evidence."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
EXPERIMENT_DIR = HERE.parent
|
|
REPO_ROOT = EXPERIMENT_DIR.parents[1]
|
|
LATEST_PATH = HERE / "latest.json"
|
|
|
|
ARMS = {"original", "qk_norm_muon"}
|
|
STAGES = {"pretrain", "sft", "dpo"}
|
|
EXPECTED_COUNTS = {
|
|
("original", "pretrain"): 7,
|
|
("original", "sft"): 8,
|
|
("original", "dpo"): 9,
|
|
("qk_norm_muon", "pretrain"): 7,
|
|
("qk_norm_muon", "sft"): 9,
|
|
("qk_norm_muon", "dpo"): 9,
|
|
}
|
|
EXPECTED_SOURCE_REVISION = "8bdc5d97d5845a8c1ac2ed56a5b8b4c0d0fb0795"
|
|
EXPECTED_DATASET_REVISION = "84983ed4dec7836d240577760c1d6be5d4cabcf9"
|
|
EXPECTED_DATASET_FILES = {
|
|
"pretrain_hq.jsonl": (
|
|
"9801b0d2210c61c2e4bc130f6dc4b3c870698a88d04af8f103c23dd5f0ce2440",
|
|
1_669_750_047,
|
|
),
|
|
"sft_512.jsonl": (
|
|
"053b7d09574e48a86232e929211434ff9e5016c6ed13312e63687dd52edcbebf",
|
|
7_531_517_862,
|
|
),
|
|
"dpo.jsonl": (
|
|
"ee934a8a455ccc99d1334d63e1254dd1d64f497fd067cfcbb71e3043f5b46768",
|
|
53_653_322,
|
|
),
|
|
}
|
|
SECRET_PATTERNS = (
|
|
re.compile(r"(?i)authorization\s*[:=]\s*bearer\s+\S+"),
|
|
re.compile(r"(?i)(?:api[_-]?key|secret)\s*[:=]\s*[A-Za-z0-9._-]{16,}"),
|
|
re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b"),
|
|
)
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def load_json(path: Path) -> dict[str, Any]:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(value, dict):
|
|
raise AssertionError(f"{path} must contain a JSON object")
|
|
return value
|
|
|
|
|
|
def resolve_relative(base: Path, relative: str) -> Path:
|
|
path = (base / relative).resolve()
|
|
if not path.is_relative_to(base.resolve()):
|
|
raise AssertionError(f"path escapes evidence root: {relative}")
|
|
return path
|
|
|
|
|
|
def check_record(path: Path, record: dict[str, Any]) -> None:
|
|
if not path.is_file():
|
|
raise AssertionError(f"missing declared file: {path}")
|
|
if path.stat().st_size != record.get("bytes"):
|
|
raise AssertionError(f"byte count mismatch: {path}")
|
|
if sha256_file(path) != record.get("sha256"):
|
|
raise AssertionError(f"SHA-256 mismatch: {path}")
|
|
|
|
|
|
def parse_response_content(content: str) -> dict[str, Any]:
|
|
stripped = content.strip()
|
|
if stripped.startswith("```"):
|
|
stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
|
|
stripped = re.sub(r"\s*```$", "", stripped)
|
|
value = json.loads(stripped)
|
|
if not isinstance(value, dict):
|
|
raise AssertionError("judge response content must decode to an object")
|
|
return value
|
|
|
|
|
|
def find_pair(
|
|
retained: dict[str, Any], arm: str, stage: str, keyword: str
|
|
) -> dict[str, str]:
|
|
cells = [
|
|
cell
|
|
for cell in retained["cells"]
|
|
if cell.get("arm") == arm and cell.get("stage") == stage
|
|
]
|
|
if len(cells) != 1:
|
|
raise AssertionError(f"missing or duplicate retained cell: {arm}/{stage}")
|
|
matches = [
|
|
pair
|
|
for pair in cells[0]["pairs"]
|
|
if keyword.lower() in pair.get("prompt", "").lower()
|
|
]
|
|
if len(matches) != 1:
|
|
raise AssertionError(f"missing or duplicate selected prompt: {arm}/{stage}/{keyword}")
|
|
return matches[0]
|
|
|
|
|
|
def validate(latest_path: Path = LATEST_PATH) -> dict[str, Any]:
|
|
latest = load_json(latest_path)
|
|
if latest.get("experiment") != "8-3" or latest.get("status") != "passed":
|
|
raise AssertionError("latest pointer is not a passed Experiment 8-3 run")
|
|
run_dir = resolve_relative(EXPERIMENT_DIR, latest["run_dir"])
|
|
manifest_path = run_dir / "manifest.json"
|
|
if sha256_file(manifest_path) != latest.get("manifest_sha256"):
|
|
raise AssertionError("latest manifest hash mismatch")
|
|
manifest = load_json(manifest_path)
|
|
if manifest.get("run_id") != latest.get("run_id"):
|
|
raise AssertionError("run ID mismatch between latest and manifest")
|
|
if manifest.get("experiment") != "8-3" or manifest.get("status") != "passed":
|
|
raise AssertionError("manifest is not a passed Experiment 8-3 run")
|
|
if manifest.get("checkpoint_policy") != "not distributed; not an acceptance artifact":
|
|
raise AssertionError("manifest checkpoint policy is incorrect")
|
|
|
|
inputs = manifest.get("inputs")
|
|
artifacts = manifest.get("artifacts")
|
|
if not isinstance(inputs, list) or len(inputs) != 5:
|
|
raise AssertionError("manifest must bind exactly five repository inputs")
|
|
if not isinstance(artifacts, list) or len(artifacts) != 5:
|
|
raise AssertionError("manifest must bind exactly five run artifacts")
|
|
for record in inputs:
|
|
check_record(resolve_relative(REPO_ROOT, record["path"]), record)
|
|
for record in artifacts:
|
|
check_record(resolve_relative(run_dir, record["path"]), record)
|
|
|
|
retained = load_json(run_dir / "retained_outputs.json")
|
|
report_record = next(
|
|
(record for record in inputs if record.get("path") == retained.get("source_report")),
|
|
None,
|
|
)
|
|
if report_record is None or retained.get("source_report_sha256") != report_record.get("sha256"):
|
|
raise AssertionError("retained source-report hash does not match the manifest input")
|
|
if retained.get("cell_count") != 6 or retained.get("output_count") != 49:
|
|
raise AssertionError("retained report must contain six cells and 49 outputs")
|
|
if set(retained.get("arms", [])) != ARMS or set(retained.get("stages", [])) != STAGES:
|
|
raise AssertionError("retained report arm/stage coverage is incomplete")
|
|
cells = retained.get("cells")
|
|
if not isinstance(cells, list) or len(cells) != 6:
|
|
raise AssertionError("retained cells are malformed")
|
|
combos: set[tuple[str, str]] = set()
|
|
for cell in cells:
|
|
combo = (cell.get("arm"), cell.get("stage"))
|
|
if combo in combos or combo not in EXPECTED_COUNTS:
|
|
raise AssertionError(f"duplicate or unexpected cell: {combo}")
|
|
combos.add(combo)
|
|
pairs = cell.get("pairs")
|
|
if not isinstance(pairs, list) or len(pairs) != EXPECTED_COUNTS[combo]:
|
|
raise AssertionError(f"wrong retained pair count for {combo}")
|
|
if any(not pair.get("prompt") or not pair.get("output") for pair in pairs):
|
|
raise AssertionError(f"empty retained prompt/output in {combo}")
|
|
if combos != set(EXPECTED_COUNTS):
|
|
raise AssertionError("not all arm/stage cells are present")
|
|
|
|
receipts_root = load_json(run_dir / "judge_receipts.json")
|
|
if receipts_root.get("credential_headers_retained") is not False:
|
|
raise AssertionError("credential header retention must be explicitly false")
|
|
calls = receipts_root.get("calls")
|
|
if not isinstance(calls, list) or len(calls) != 8:
|
|
raise AssertionError("exactly eight raw judge calls are required")
|
|
response_ids: set[str] = set()
|
|
normalized_rows: dict[str, Any] = {}
|
|
for expected_case_id, call in enumerate(calls, start=1):
|
|
if call.get("case_id") != expected_case_id or call.get("http_status") != 200:
|
|
raise AssertionError("judge calls must be successful and ordered by case ID")
|
|
if call.get("credential_headers_retained") is not False:
|
|
raise AssertionError("per-call credential retention boundary is missing")
|
|
response_id = call.get("response_id")
|
|
if not isinstance(response_id, str) or not response_id or response_id in response_ids:
|
|
raise AssertionError("judge response IDs must be present and unique")
|
|
response_ids.add(response_id)
|
|
if call.get("latency_ms", 0) <= 0 or call.get("usage", {}).get("total_tokens", 0) <= 0:
|
|
raise AssertionError("judge usage and positive latency must be retained")
|
|
raw_response = call.get("response", {})
|
|
if raw_response.get("id") != response_id or raw_response.get("usage") != call.get("usage"):
|
|
raise AssertionError("copied response ID/usage does not match raw response")
|
|
try:
|
|
content = raw_response["choices"][0]["message"]["content"]
|
|
except (KeyError, IndexError, TypeError) as exc:
|
|
raise AssertionError("raw judge response is missing message content") from exc
|
|
judgment = parse_response_content(content)
|
|
if judgment != call.get("judgment"):
|
|
raise AssertionError("normalized judgment does not match raw response content")
|
|
if str(judgment.get("case_id")) != str(expected_case_id):
|
|
raise AssertionError("raw judgment has the wrong case ID")
|
|
if set(judgment.get("candidates", {})) != {"A", "B"}:
|
|
raise AssertionError("judge judgment must score A and B")
|
|
if judgment.get("winner") not in {"A", "B", "tie"}:
|
|
raise AssertionError("judge winner is invalid")
|
|
|
|
mapping = call.get("blind_map")
|
|
if not isinstance(mapping, dict) or set(mapping) != {"A", "B"} or set(mapping.values()) != ARMS:
|
|
raise AssertionError("blind mapping must cover both arms")
|
|
request = call.get("request")
|
|
request_text = json.dumps(request, ensure_ascii=False).lower()
|
|
if "qk_norm_muon" in request_text or '"original"' in request_text:
|
|
raise AssertionError("judge request leaks an arm identity")
|
|
try:
|
|
user_payload = json.loads(request["messages"][1]["content"])
|
|
request_candidates = user_payload["candidates"]
|
|
except (KeyError, IndexError, TypeError, json.JSONDecodeError) as exc:
|
|
raise AssertionError("judge request is missing structured candidates") from exc
|
|
if user_payload.get("case_id") != expected_case_id:
|
|
raise AssertionError("judge request case ID mismatch")
|
|
for label, arm in mapping.items():
|
|
retained_pair = find_pair(retained, arm, call["stage"], call["keyword"])
|
|
expected_candidate = {
|
|
"historical_prompt": retained_pair["prompt"],
|
|
"historical_output": retained_pair["output"],
|
|
}
|
|
if request_candidates.get(label) != expected_candidate:
|
|
raise AssertionError("raw judge request is not bound to the retained output")
|
|
normalized_rows[str(expected_case_id)] = {
|
|
mapping[label]: score for label, score in judgment["candidates"].items()
|
|
}
|
|
|
|
contract = load_json(run_dir / "reproduction_contract.json")
|
|
future = contract.get("future_reproduction", {})
|
|
source = future.get("source", {})
|
|
dataset = future.get("dataset", {})
|
|
if source.get("revision") != EXPECTED_SOURCE_REVISION:
|
|
raise AssertionError("frozen MiniMind source revision mismatch")
|
|
source_hashes = source.get("file_sha256")
|
|
if not isinstance(source_hashes, dict) or len(source_hashes) < 12:
|
|
raise AssertionError("frozen source file hashes are incomplete")
|
|
if any(not re.fullmatch(r"[0-9a-f]{64}", value) for value in source_hashes.values()):
|
|
raise AssertionError("invalid frozen source SHA-256")
|
|
if dataset.get("revision") != EXPECTED_DATASET_REVISION:
|
|
raise AssertionError("frozen dataset revision mismatch")
|
|
dataset_files = dataset.get("files", {})
|
|
for name, (expected_hash, expected_bytes) in EXPECTED_DATASET_FILES.items():
|
|
record = dataset_files.get(name, {})
|
|
if record.get("lfs_sha256") != expected_hash or record.get("bytes") != expected_bytes:
|
|
raise AssertionError(f"frozen dataset file mismatch: {name}")
|
|
if len(future.get("commands", {})) != 6:
|
|
raise AssertionError("all six reproduction commands are required")
|
|
boundary = contract.get("historical_evidence_boundary", {})
|
|
for key in (
|
|
"historical_source_revision_retained",
|
|
"historical_dataset_hashes_retained",
|
|
"historical_checkpoint_hashes_retained",
|
|
"historical_stepwise_training_logs_retained",
|
|
):
|
|
if boundary.get(key) is not False:
|
|
raise AssertionError(f"historical provenance boundary is not explicit: {key}")
|
|
mechanisms = contract.get("model_and_training", {}).get("source_verified_mechanisms", {})
|
|
if not mechanisms or not all(value is True for value in mechanisms.values()):
|
|
raise AssertionError("source mechanism assertions are incomplete")
|
|
policy = contract.get("checkpoint_policy", {})
|
|
if policy.get("distributed_with_book") is not False or policy.get("acceptance_artifact") is not False:
|
|
raise AssertionError("checkpoint policy does not match the book contract")
|
|
|
|
summary = load_json(run_dir / "summary.json")
|
|
if summary.get("status") != "passed" or summary.get("acceptance", {}).get("passed") is not True:
|
|
raise AssertionError("summary acceptance did not pass")
|
|
acceptance = summary["acceptance"]
|
|
if not all(value is True for key, value in acceptance.items() if key != "passed"):
|
|
failed = [key for key, value in acceptance.items() if key != "passed" and value is not True]
|
|
raise AssertionError(f"required acceptance gates failed: {failed}")
|
|
if summary.get("per_case_arm_scores") != normalized_rows:
|
|
raise AssertionError("summary scores do not match raw judge responses")
|
|
findings = summary.get("scientific_findings", {})
|
|
if not isinstance(findings.get("blind_judge_prefers_qk_norm_muon_overall"), bool):
|
|
raise AssertionError("comparative scientific finding is missing")
|
|
if findings.get("reported_loss_comparison_retained_but_not_independently_recomputed") is not True:
|
|
raise AssertionError("loss-evidence qualification is missing")
|
|
|
|
for record in artifacts:
|
|
path = resolve_relative(run_dir, record["path"])
|
|
if path.suffix not in {".json", ".md"}:
|
|
continue
|
|
text = path.read_text(encoding="utf-8")
|
|
for pattern in SECRET_PATTERNS:
|
|
if pattern.search(text):
|
|
raise AssertionError(f"possible credential in retained artifact: {path.name}")
|
|
|
|
return {
|
|
"experiment": "8-3",
|
|
"run_id": latest["run_id"],
|
|
"status": "passed",
|
|
"inputs_verified": len(inputs),
|
|
"artifacts_verified": len(artifacts),
|
|
"judge_receipts_verified": len(calls),
|
|
"outputs_verified": retained["output_count"],
|
|
"manifest_sha256": latest["manifest_sha256"],
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--latest", type=Path, default=LATEST_PATH)
|
|
args = parser.parse_args()
|
|
result = validate(args.latest.resolve())
|
|
print(json.dumps(result, indent=2, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|