ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"experiment": "8-5",
|
||||
"manifest_sha256": "89d180e515fc664d3c52e057ad72f16fef82a455439e53a7ef9e3d16273e5819",
|
||||
"run_dir": "validation/runs/exp8-5-training-report-20260731-v1",
|
||||
"run_id": "exp8-5-training-report-20260731-v1",
|
||||
"schema_version": "exp8-5-latest-v1",
|
||||
"status": "passed"
|
||||
}
|
||||
@@ -0,0 +1,780 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the canonical, checkpoint-free evidence package for Experiment 8-5.
|
||||
|
||||
The historical RTX 4090 run is retained as a raw terminal transcript in
|
||||
``model_eval_results.md``. This tool does not pretend to rerun that GPU job.
|
||||
It extracts the fifteen saved generations, submits five stage-blind comparison
|
||||
tasks to an independent judge, and binds the report, current reproduction
|
||||
sources, frozen upstream revisions, receipts, findings, and limitations into a
|
||||
content-hashed manifest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
EXPERIMENT_DIR = HERE.parent
|
||||
REPO_ROOT = EXPERIMENT_DIR.parents[1]
|
||||
REPORT_PATH = EXPERIMENT_DIR / "model_eval_results.md"
|
||||
RUNS_DIR = HERE / "runs"
|
||||
LATEST_PATH = HERE / "latest.json"
|
||||
|
||||
DEFAULT_RUN_ID = "exp8-5-training-report-20260731-v1"
|
||||
DEFAULT_ENDPOINT = "https://ark.cn-beijing.volces.com/api/v3/chat/completions"
|
||||
DEFAULT_MODEL = "doubao-seed-1-6-250615"
|
||||
BLIND_SEED = 750731
|
||||
ENGLISH_RETENTION_TOLERANCE = 1.0
|
||||
|
||||
BASE_REVISION = "9ea1b83f5ced5663c5fa89c300fe59f9bdcd2b10"
|
||||
WIKIPEDIA_REVISION = "b04c8d1ceb2f5cd4588862100d08de323dccfbaa"
|
||||
ALPACA_REVISION = "f38ae19cf673363d74fab6217de46c1b9c3150d4"
|
||||
|
||||
TEST_RE = re.compile(r"^Test ([1-5]):\s*(.+)$")
|
||||
BOX_RE = re.compile(r"^┌─ (BASELINE|PRETRAINED|FINETUNED) MODEL\b")
|
||||
CLOSE_RE = re.compile(r"^└─+")
|
||||
STAGES = ("baseline", "pretrained", "finetuned")
|
||||
LABELS = ("A", "B", "C")
|
||||
|
||||
PROMPTS = {
|
||||
1: {
|
||||
"language": "korean",
|
||||
"task": "Write the opening of a Korean Wikipedia article about artificial intelligence.",
|
||||
},
|
||||
2: {
|
||||
"language": "english",
|
||||
"task": "Write the opening of an English Wikipedia article about artificial intelligence.",
|
||||
},
|
||||
3: {
|
||||
"language": "korean",
|
||||
"task": "한국의 전통 음식인 김치에 대해 설명하세요.",
|
||||
},
|
||||
4: {
|
||||
"language": "korean",
|
||||
"task": "대한민국의 수도인 서울에 대해 간단히 소개해주세요.",
|
||||
},
|
||||
5: {
|
||||
"language": "english",
|
||||
"task": "Explain Thanksgiving turkey, a traditional American food.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def canonical_json_bytes(value: Any) -> bytes:
|
||||
return (json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode()
|
||||
|
||||
|
||||
def write_json(path: Path, value: Any) -> None:
|
||||
path.write_bytes(canonical_json_bytes(value))
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def parse_retained_outputs(report_path: Path = REPORT_PATH) -> dict[str, Any]:
|
||||
"""Extract the five-by-three raw comparison matrix from the saved report."""
|
||||
|
||||
lines = report_path.read_text(encoding="utf-8").splitlines()
|
||||
tests: dict[int, dict[str, Any]] = {}
|
||||
current_test: int | None = None
|
||||
current_stage: str | None = None
|
||||
body: list[str] = []
|
||||
|
||||
for line in lines:
|
||||
test_match = TEST_RE.match(line)
|
||||
if test_match:
|
||||
current_test = int(test_match.group(1))
|
||||
if current_test in tests:
|
||||
raise ValueError(f"duplicate raw Test {current_test}")
|
||||
tests[current_test] = {
|
||||
"test_id": current_test,
|
||||
"name": test_match.group(2).strip(),
|
||||
"language": PROMPTS[current_test]["language"],
|
||||
"task": PROMPTS[current_test]["task"],
|
||||
"outputs": {},
|
||||
}
|
||||
continue
|
||||
|
||||
box_match = BOX_RE.match(line)
|
||||
if box_match:
|
||||
if current_test is None:
|
||||
raise ValueError("model output box appeared before a raw Test heading")
|
||||
if current_stage is not None:
|
||||
raise ValueError("nested model output boxes")
|
||||
current_stage = box_match.group(1).lower()
|
||||
body = []
|
||||
continue
|
||||
|
||||
if current_stage is None:
|
||||
continue
|
||||
|
||||
if CLOSE_RE.match(line):
|
||||
output = "\n".join(body).strip()
|
||||
if not output:
|
||||
raise ValueError(f"empty {current_stage} output in Test {current_test}")
|
||||
outputs = tests[current_test]["outputs"]
|
||||
if current_stage in outputs:
|
||||
raise ValueError(f"duplicate {current_stage} output in Test {current_test}")
|
||||
outputs[current_stage] = output
|
||||
current_stage = None
|
||||
body = []
|
||||
continue
|
||||
|
||||
if line == "│":
|
||||
body.append("")
|
||||
elif line.startswith("│ "):
|
||||
body.append(line[2:])
|
||||
elif line.startswith("│"):
|
||||
body.append(line[1:].lstrip())
|
||||
else:
|
||||
# The historical terminal capture wrapped a few long lines without
|
||||
# repeating the box prefix. Preserve those bytes as output text.
|
||||
body.append(line)
|
||||
|
||||
if current_stage is not None:
|
||||
raise ValueError("unterminated model output box")
|
||||
if set(tests) != set(PROMPTS):
|
||||
raise ValueError(f"expected Tests 1-5, found {sorted(tests)}")
|
||||
|
||||
for test_id, test in tests.items():
|
||||
if set(test["outputs"]) != set(STAGES):
|
||||
raise ValueError(
|
||||
f"Test {test_id} expected stages {STAGES}, found {sorted(test['outputs'])}"
|
||||
)
|
||||
|
||||
ordered = [tests[test_id] for test_id in sorted(tests)]
|
||||
return {
|
||||
"schema_version": "exp8-5-retained-outputs-v1",
|
||||
"source_report": str(REPORT_PATH.relative_to(REPO_ROOT)),
|
||||
"source_report_sha256": sha256_file(report_path),
|
||||
"test_count": len(ordered),
|
||||
"output_count": sum(len(test["outputs"]) for test in ordered),
|
||||
"tests": ordered,
|
||||
}
|
||||
|
||||
|
||||
def blind_mapping(test_id: int) -> dict[str, str]:
|
||||
stages = list(STAGES)
|
||||
random.Random(BLIND_SEED + test_id).shuffle(stages)
|
||||
return dict(zip(LABELS, stages, strict=True))
|
||||
|
||||
|
||||
def judge_payload(test: dict[str, Any], mapping: dict[str, str], model: str) -> dict[str, Any]:
|
||||
candidates = {
|
||||
label: test["outputs"][stage]
|
||||
for label, stage in mapping.items()
|
||||
}
|
||||
rubric = {
|
||||
"language_fluency": "0 unreadable; 3 understandable with defects; 5 native-quality and coherent",
|
||||
"instruction_following": "0 ignores the task; 3 partly satisfies it; 5 directly and fully satisfies it",
|
||||
"factuality": "0 dominated by falsehoods; 3 mixed/minor errors; 5 accurate with no material error",
|
||||
}
|
||||
expected_shape = {
|
||||
"test_id": test["test_id"],
|
||||
"language": test["language"],
|
||||
"candidates": {
|
||||
label: {
|
||||
"language_fluency": "number 0-5",
|
||||
"instruction_following": "number 0-5",
|
||||
"factuality": "number 0-5",
|
||||
"factual_errors": ["specific error, empty only if none"],
|
||||
"rationale": "short evidence-based explanation",
|
||||
}
|
||||
for label in LABELS
|
||||
},
|
||||
"ranking": ["best label", "middle label", "worst label"],
|
||||
}
|
||||
user_content = {
|
||||
"test_id": test["test_id"],
|
||||
"language": test["language"],
|
||||
"task": test["task"],
|
||||
"rubric": rubric,
|
||||
"candidates": candidates,
|
||||
"required_json_shape": expected_shape,
|
||||
}
|
||||
return {
|
||||
"model": model,
|
||||
"temperature": 0,
|
||||
"response_format": {"type": "json_object"},
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are an independent bilingual Korean/English evaluator. "
|
||||
"The candidates are deliberately anonymous; do not infer model identity or training stage. "
|
||||
"Score only the supplied text. Identify concrete factual errors, especially invented food "
|
||||
"ingredients or preparation claims. Return one JSON object only, with every requested field."
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": json.dumps(user_content, ensure_ascii=False, sort_keys=True),
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def extract_json_object(content: str) -> dict[str, Any]:
|
||||
stripped = content.strip()
|
||||
if stripped.startswith("```"):
|
||||
stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
|
||||
stripped = re.sub(r"\s*```$", "", stripped)
|
||||
parsed = json.loads(stripped)
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError("judge content must decode to an object")
|
||||
return parsed
|
||||
|
||||
|
||||
def validate_judgment(judgment: dict[str, Any], test: dict[str, Any]) -> None:
|
||||
if judgment.get("test_id") != test["test_id"]:
|
||||
raise ValueError("judge returned the wrong test_id")
|
||||
if judgment.get("language") != test["language"]:
|
||||
raise ValueError("judge returned the wrong language")
|
||||
candidates = judgment.get("candidates")
|
||||
if not isinstance(candidates, dict) or set(candidates) != set(LABELS):
|
||||
raise ValueError("judge must score exactly candidates A, B, and C")
|
||||
for label in LABELS:
|
||||
row = candidates[label]
|
||||
if not isinstance(row, dict):
|
||||
raise ValueError(f"candidate {label} score must be an object")
|
||||
for metric in ("language_fluency", "instruction_following", "factuality"):
|
||||
score = row.get(metric)
|
||||
if not isinstance(score, (int, float)) or isinstance(score, bool) or not 0 <= score <= 5:
|
||||
raise ValueError(f"candidate {label} has invalid {metric}: {score!r}")
|
||||
errors = row.get("factual_errors")
|
||||
if not isinstance(errors, list) or not all(isinstance(item, str) for item in errors):
|
||||
raise ValueError(f"candidate {label} factual_errors must be a list of strings")
|
||||
if not isinstance(row.get("rationale"), str) or not row["rationale"].strip():
|
||||
raise ValueError(f"candidate {label} rationale is missing")
|
||||
ranking = judgment.get("ranking")
|
||||
if not isinstance(ranking, list) or set(ranking) != set(LABELS) or len(ranking) != 3:
|
||||
raise ValueError("judge ranking must contain A, B, and C exactly once")
|
||||
|
||||
|
||||
def call_judge(
|
||||
test: dict[str, Any],
|
||||
*,
|
||||
endpoint: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
timeout: float,
|
||||
) -> dict[str, Any]:
|
||||
mapping = blind_mapping(test["test_id"])
|
||||
payload = judge_payload(test, mapping, model)
|
||||
request = urllib.request.Request(
|
||||
endpoint,
|
||||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
method="POST",
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
response_body = response.read()
|
||||
http_status = response.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
raise RuntimeError(f"judge HTTP {exc.code}: {body[:500]}") from exc
|
||||
latency_ms = round((time.perf_counter() - started) * 1000, 3)
|
||||
raw_response = json.loads(response_body)
|
||||
try:
|
||||
content = raw_response["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise ValueError("judge response has no choices[0].message.content") from exc
|
||||
judgment = extract_json_object(content)
|
||||
validate_judgment(judgment, test)
|
||||
|
||||
response_id = raw_response.get("id")
|
||||
usage = raw_response.get("usage")
|
||||
if not isinstance(response_id, str) or not response_id:
|
||||
raise ValueError("judge response has no response ID")
|
||||
if not isinstance(usage, dict) or not isinstance(usage.get("total_tokens"), int):
|
||||
raise ValueError("judge response has no complete usage object")
|
||||
|
||||
return {
|
||||
"test_id": test["test_id"],
|
||||
"provider": "ark",
|
||||
"endpoint": endpoint,
|
||||
"credential_env": "ARK_API_KEY",
|
||||
"blind_seed": BLIND_SEED,
|
||||
"blind_map": mapping,
|
||||
"request": payload,
|
||||
"http_status": http_status,
|
||||
"response": raw_response,
|
||||
"response_id": response_id,
|
||||
"usage": usage,
|
||||
"latency_ms": latency_ms,
|
||||
"judgment": judgment,
|
||||
}
|
||||
|
||||
|
||||
def reproduction_contract() -> dict[str, Any]:
|
||||
pin_note = (
|
||||
"This immutable revision is the frozen reproduction contract selected on 2026-07-31. "
|
||||
"The historical run did not retain its resolved upstream commit, so this is not claimed "
|
||||
"to be the exact historical revision."
|
||||
)
|
||||
return {
|
||||
"schema_version": "exp8-5-reproduction-contract-v1",
|
||||
"experiment": "8-5",
|
||||
"historical_evidence_boundary": {
|
||||
"historical_training_executed": True,
|
||||
"raw_three_stage_evaluation_retained": True,
|
||||
"historical_upstream_revisions_retained": False,
|
||||
"historical_checkpoint_hashes_retained": False,
|
||||
"claim": (
|
||||
"The retained terminal report proves a three-stage evaluation ran on the reported RTX 4090 "
|
||||
"software stack. It does not prove the byte identity of the historical adapters or upstream data."
|
||||
),
|
||||
},
|
||||
"upstream_revisions": {
|
||||
"base_model": {
|
||||
"repository": "unsloth/mistral-7b-v0.3",
|
||||
"revision": BASE_REVISION,
|
||||
"note": pin_note,
|
||||
},
|
||||
"continued_pretraining_dataset": {
|
||||
"repository": "wikimedia/wikipedia",
|
||||
"configuration": "20231101.ko",
|
||||
"revision": WIKIPEDIA_REVISION,
|
||||
"note": pin_note,
|
||||
},
|
||||
"instruction_dataset": {
|
||||
"repository": "FreedomIntelligence/alpaca-gpt4-korean",
|
||||
"revision": ALPACA_REVISION,
|
||||
"note": pin_note,
|
||||
},
|
||||
},
|
||||
"training": {
|
||||
"model_loading": {"max_sequence_length": 2048, "load_in_4bit": True},
|
||||
"lora": {
|
||||
"rank": 128,
|
||||
"alpha": 32,
|
||||
"dropout": 0,
|
||||
"bias": "none",
|
||||
"use_rslora": True,
|
||||
"random_state": 3407,
|
||||
"gradient_checkpointing": "unsloth",
|
||||
"target_modules": [
|
||||
"q_proj",
|
||||
"k_proj",
|
||||
"v_proj",
|
||||
"o_proj",
|
||||
"gate_proj",
|
||||
"up_proj",
|
||||
"down_proj",
|
||||
"embed_tokens",
|
||||
"lm_head",
|
||||
],
|
||||
},
|
||||
"continued_pretraining": {
|
||||
"dataset_fraction": 0.05,
|
||||
"epochs": 1,
|
||||
"max_steps": -1,
|
||||
"batch_size": 2,
|
||||
"gradient_accumulation_steps": 8,
|
||||
"learning_rate": 5e-5,
|
||||
"embedding_learning_rate": 1e-5,
|
||||
"warmup_steps": 10,
|
||||
"warmup_ratio": 0.1,
|
||||
"optimizer": "adamw_8bit",
|
||||
"weight_decay": 0.01,
|
||||
"scheduler": "linear",
|
||||
"trainer_seed": 42,
|
||||
"dataset_split_seed": "not explicitly recorded by the historical script",
|
||||
},
|
||||
"instruction_sft": {
|
||||
"epochs": 2,
|
||||
"max_steps": -1,
|
||||
"batch_size": 2,
|
||||
"gradient_accumulation_steps": 8,
|
||||
"learning_rate": 5e-5,
|
||||
"embedding_learning_rate": 1e-5,
|
||||
"warmup_steps": 10,
|
||||
"warmup_ratio": 0.1,
|
||||
"optimizer": "adamw_8bit",
|
||||
"weight_decay": 0.0,
|
||||
"scheduler": "linear",
|
||||
"trainer_seed": 42,
|
||||
},
|
||||
},
|
||||
"evaluation": {
|
||||
"stages": list(STAGES),
|
||||
"test_count": 5,
|
||||
"output_count": 15,
|
||||
"max_new_tokens": 150,
|
||||
"temperature": 0.3,
|
||||
"do_sample": True,
|
||||
"historical_generation_seed": "not retained",
|
||||
},
|
||||
"historical_environment_from_report": {
|
||||
"gpu": "NVIDIA GeForce RTX 4090",
|
||||
"gpu_memory_gb": 23.647,
|
||||
"platform": "Linux",
|
||||
"torch": "2.8.0+cu128",
|
||||
"cuda_compute_capability": "8.9",
|
||||
"cuda_toolkit": "12.8",
|
||||
"unsloth": "2025.10.4",
|
||||
"transformers": "4.56.2",
|
||||
"triton": "3.4.0",
|
||||
"xformers": "0.0.32.post2",
|
||||
},
|
||||
"checkpoint_policy": {
|
||||
"distributed_with_book": False,
|
||||
"acceptance_artifact": False,
|
||||
"required_artifact": "reproducible evidence-backed training report",
|
||||
"reason": "Training adapters are intentionally local and are not distributed to readers.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def mean(values: list[float]) -> float:
|
||||
return round(sum(values) / len(values), 4)
|
||||
|
||||
|
||||
def summarize(
|
||||
retained: dict[str, Any], receipts: list[dict[str, Any]], contract: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
by_test = {test["test_id"]: test for test in retained["tests"]}
|
||||
stage_rows: dict[int, dict[str, dict[str, Any]]] = {}
|
||||
for receipt in receipts:
|
||||
reverse = {label: stage for label, stage in receipt["blind_map"].items()}
|
||||
stage_rows[receipt["test_id"]] = {
|
||||
reverse[label]: score
|
||||
for label, score in receipt["judgment"]["candidates"].items()
|
||||
}
|
||||
|
||||
metrics = ("language_fluency", "instruction_following", "factuality")
|
||||
stage_averages: dict[str, dict[str, Any]] = {}
|
||||
for stage in STAGES:
|
||||
korean_rows = [
|
||||
stage_rows[test_id][stage]
|
||||
for test_id in (1, 3, 4)
|
||||
]
|
||||
english_rows = [
|
||||
stage_rows[test_id][stage]
|
||||
for test_id in (2, 5)
|
||||
]
|
||||
stage_averages[stage] = {
|
||||
"korean": {
|
||||
metric: mean([float(row[metric]) for row in korean_rows])
|
||||
for metric in metrics
|
||||
},
|
||||
"english": {
|
||||
metric: mean([float(row[metric]) for row in english_rows])
|
||||
for metric in metrics
|
||||
},
|
||||
}
|
||||
stage_averages[stage]["korean"]["overall"] = mean(
|
||||
[float(row[metric]) for row in korean_rows for metric in metrics]
|
||||
)
|
||||
stage_averages[stage]["english"]["overall"] = mean(
|
||||
[float(row[metric]) for row in english_rows for metric in metrics]
|
||||
)
|
||||
|
||||
baseline_korean = stage_averages["baseline"]["korean"]["overall"]
|
||||
final_korean = stage_averages["finetuned"]["korean"]["overall"]
|
||||
baseline_english = stage_averages["baseline"]["english"]["overall"]
|
||||
final_english = stage_averages["finetuned"]["english"]["overall"]
|
||||
english_drop = round(baseline_english - final_english, 4)
|
||||
kimchi_errors = stage_rows[3]["finetuned"]["factual_errors"]
|
||||
|
||||
findings = {
|
||||
"korean_gain_observed": final_korean > baseline_korean,
|
||||
"korean_gain": round(final_korean - baseline_korean, 4),
|
||||
"english_retention_tolerance": ENGLISH_RETENTION_TOLERANCE,
|
||||
"english_drop": english_drop,
|
||||
"english_retention_within_tolerance": english_drop <= ENGLISH_RETENTION_TOLERANCE,
|
||||
"kimchi_factual_failure_observed": bool(kimchi_errors),
|
||||
"kimchi_finetuned_factual_errors": kimchi_errors,
|
||||
}
|
||||
execution_gates = {
|
||||
"raw_report_hashed": bool(retained["source_report_sha256"]),
|
||||
"exactly_five_tests": retained["test_count"] == 5,
|
||||
"exactly_fifteen_outputs": retained["output_count"] == 15,
|
||||
"all_three_stages_retained": all(
|
||||
set(test["outputs"]) == set(STAGES) for test in retained["tests"]
|
||||
),
|
||||
"five_independent_blind_judgments": len(receipts) == 5,
|
||||
"judge_response_ids_usage_and_latency_retained": all(
|
||||
receipt["response_id"]
|
||||
and receipt["usage"].get("total_tokens", 0) > 0
|
||||
and receipt["latency_ms"] > 0
|
||||
for receipt in receipts
|
||||
),
|
||||
"training_and_evaluation_sources_declared": True,
|
||||
"immutable_future_reproduction_revisions_frozen": all(
|
||||
contract["upstream_revisions"][key]["revision"]
|
||||
for key in (
|
||||
"base_model",
|
||||
"continued_pretraining_dataset",
|
||||
"instruction_dataset",
|
||||
)
|
||||
),
|
||||
"historical_revision_boundary_explicit": (
|
||||
contract["historical_evidence_boundary"]["historical_upstream_revisions_retained"]
|
||||
is False
|
||||
),
|
||||
"checkpoints_not_an_acceptance_artifact": (
|
||||
contract["checkpoint_policy"]["acceptance_artifact"] is False
|
||||
),
|
||||
# Scientific outcomes are reported, not promoted into evidence-completeness
|
||||
# gates. A real negative result still completes the prescribed comparison.
|
||||
"korean_gain_comparison_completed": isinstance(findings["korean_gain"], float),
|
||||
"english_retention_comparison_completed": isinstance(findings["english_drop"], float),
|
||||
"kimchi_failure_explicitly_reported": findings["kimchi_factual_failure_observed"],
|
||||
}
|
||||
passed = all(execution_gates.values())
|
||||
return {
|
||||
"schema_version": "exp8-5-summary-v1",
|
||||
"experiment": "8-5",
|
||||
"status": "passed" if passed else "failed",
|
||||
"judge": {
|
||||
"provider": "ark",
|
||||
"model": receipts[0]["request"]["model"],
|
||||
"calls": len(receipts),
|
||||
"response_ids": [receipt["response_id"] for receipt in receipts],
|
||||
"total_tokens": sum(receipt["usage"]["total_tokens"] for receipt in receipts),
|
||||
"total_latency_ms": round(sum(receipt["latency_ms"] for receipt in receipts), 3),
|
||||
"blind_seed": BLIND_SEED,
|
||||
},
|
||||
"stage_averages": stage_averages,
|
||||
"per_test_stage_scores": stage_rows,
|
||||
"scientific_findings": findings,
|
||||
"acceptance": {**execution_gates, "passed": passed},
|
||||
"limitations": [
|
||||
"The historical adapters/checkpoints are intentionally not distributed and were not re-created.",
|
||||
"The exact historical upstream revisions and generation RNG seed were not retained.",
|
||||
"The frozen upstream revisions are a future reproduction contract, not historical provenance.",
|
||||
"The retained evaluation has five prompts and one sampled generation per stage/prompt.",
|
||||
],
|
||||
"test_names": {str(test_id): by_test[test_id]["name"] for test_id in sorted(by_test)},
|
||||
}
|
||||
|
||||
|
||||
def render_report(summary: dict[str, Any]) -> str:
|
||||
averages = summary["stage_averages"]
|
||||
findings = summary["scientific_findings"]
|
||||
rows = []
|
||||
for stage in STAGES:
|
||||
rows.append(
|
||||
f"| {stage} | {averages[stage]['korean']['overall']:.4f} | "
|
||||
f"{averages[stage]['english']['overall']:.4f} |"
|
||||
)
|
||||
kimchi = "; ".join(findings["kimchi_finetuned_factual_errors"])
|
||||
return "\n".join(
|
||||
[
|
||||
"# Experiment 8-5 retained-training-report audit",
|
||||
"",
|
||||
"## Result",
|
||||
"",
|
||||
f"Status: **{summary['status']}**. The historical RTX 4090 report contains all five "
|
||||
"prompts across the baseline, continued-pretrained, and instruction-tuned stages. "
|
||||
"An independent stage-blind ARK judge scored the exact 15 retained outputs.",
|
||||
"",
|
||||
"| Stage | Korean mean (0-5) | English mean (0-5) |",
|
||||
"| --- | ---: | ---: |",
|
||||
*rows,
|
||||
"",
|
||||
f"Observed Korean gain, final minus baseline: **{findings['korean_gain']:+.4f}**.",
|
||||
f"Observed English drop, baseline minus final: **{findings['english_drop']:+.4f}** "
|
||||
f"(declared tolerance: {findings['english_retention_tolerance']:.1f}).",
|
||||
(
|
||||
"The final English score is within the declared tolerance."
|
||||
if findings["english_retention_within_tolerance"]
|
||||
else "The final English score is outside the declared tolerance; the historical retention "
|
||||
"claim is not supported by this blind audit."
|
||||
),
|
||||
"",
|
||||
"## Material negative result",
|
||||
"",
|
||||
"The final model's Korean is more fluent, but the kimchi answer remains factually unsafe. "
|
||||
f"The blind judge identified: {kimchi}",
|
||||
"",
|
||||
"## Provenance boundary",
|
||||
"",
|
||||
"The raw terminal report records the historical GPU/software identity and generated text, "
|
||||
"but not adapter hashes, the exact resolved upstream commits, or the sampling seed. The "
|
||||
"immutable Hugging Face revisions in `reproduction_contract.json` were selected on "
|
||||
"2026-07-31 for future reproduction and are not represented as the historical revisions.",
|
||||
"",
|
||||
"Checkpoints are intentionally local and are not an acceptance artifact. The accepted "
|
||||
"book artifact is this reproducible, evidence-backed training report.",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def input_record(path: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"path": str(path.relative_to(REPO_ROOT)),
|
||||
"sha256": sha256_file(path),
|
||||
"bytes": path.stat().st_size,
|
||||
}
|
||||
|
||||
|
||||
def artifact_record(path: Path, run_dir: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"path": str(path.relative_to(run_dir)),
|
||||
"sha256": sha256_file(path),
|
||||
"bytes": path.stat().st_size,
|
||||
}
|
||||
|
||||
|
||||
def build_manifest(run_id: str, run_dir: Path, summary: dict[str, Any]) -> dict[str, Any]:
|
||||
inputs = [
|
||||
input_record(REPORT_PATH),
|
||||
input_record(EXPERIMENT_DIR / "continued-pretrain.py"),
|
||||
input_record(EXPERIMENT_DIR / "compare_models.py"),
|
||||
input_record(EXPERIMENT_DIR / "evaluate_model.py"),
|
||||
input_record(HERE / "run_report_audit.py"),
|
||||
input_record(HERE / "validate_evidence.py"),
|
||||
]
|
||||
artifact_paths = [
|
||||
run_dir / "retained_outputs.json",
|
||||
run_dir / "reproduction_contract.json",
|
||||
run_dir / "judge_receipts.json",
|
||||
run_dir / "summary.json",
|
||||
run_dir / "report.md",
|
||||
]
|
||||
return {
|
||||
"schema_version": "exp8-5-manifest-v1",
|
||||
"experiment": "8-5",
|
||||
"run_id": run_id,
|
||||
"created_at": utc_now(),
|
||||
"status": summary["status"],
|
||||
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
|
||||
"inputs": inputs,
|
||||
"artifacts": [artifact_record(path, run_dir) for path in artifact_paths],
|
||||
"acceptance": summary["acceptance"],
|
||||
"checkpoint_policy": "not distributed; not an acceptance artifact",
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--run-id", default=DEFAULT_RUN_ID)
|
||||
parser.add_argument("--endpoint", default=os.getenv("ARK_BASE_URL", DEFAULT_ENDPOINT))
|
||||
parser.add_argument("--model", default=os.getenv("ARK_MODEL", DEFAULT_MODEL))
|
||||
parser.add_argument("--api-key-env", default="ARK_API_KEY")
|
||||
parser.add_argument("--timeout", type=float, default=180.0)
|
||||
parser.add_argument("--concurrency", type=int, default=5)
|
||||
parser.add_argument(
|
||||
"--refresh-manifest",
|
||||
action="store_true",
|
||||
help="Rehash an existing run after pre-commit source-only corrections; makes no provider call.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if not re.fullmatch(r"[A-Za-z0-9._-]+", args.run_id):
|
||||
raise SystemExit("run ID may contain only letters, digits, dot, underscore, and hyphen")
|
||||
|
||||
run_dir = RUNS_DIR / args.run_id
|
||||
if args.refresh_manifest:
|
||||
if not run_dir.is_dir():
|
||||
raise SystemExit(f"cannot refresh missing run: {run_dir}")
|
||||
summary = json.loads((run_dir / "summary.json").read_text(encoding="utf-8"))
|
||||
manifest = build_manifest(args.run_id, run_dir, summary)
|
||||
write_json(run_dir / "manifest.json", manifest)
|
||||
latest = {
|
||||
"schema_version": "exp8-5-latest-v1",
|
||||
"experiment": "8-5",
|
||||
"run_id": args.run_id,
|
||||
"status": summary["status"],
|
||||
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
|
||||
"manifest_sha256": sha256_file(run_dir / "manifest.json"),
|
||||
}
|
||||
write_json(LATEST_PATH, latest)
|
||||
print(json.dumps(latest, indent=2, sort_keys=True))
|
||||
return 0
|
||||
if run_dir.exists():
|
||||
raise SystemExit(f"refusing to overwrite existing run: {run_dir}")
|
||||
api_key = os.getenv(args.api_key_env)
|
||||
if not api_key:
|
||||
raise SystemExit(f"{args.api_key_env} is required for the independent judge")
|
||||
|
||||
retained = parse_retained_outputs()
|
||||
if not 1 <= args.concurrency <= 5:
|
||||
raise SystemExit("concurrency must be between 1 and 5")
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as executor:
|
||||
receipts = list(
|
||||
executor.map(
|
||||
lambda test: call_judge(
|
||||
test,
|
||||
endpoint=args.endpoint,
|
||||
model=args.model,
|
||||
api_key=api_key,
|
||||
timeout=args.timeout,
|
||||
),
|
||||
retained["tests"],
|
||||
)
|
||||
)
|
||||
contract = reproduction_contract()
|
||||
summary = summarize(retained, receipts, contract)
|
||||
if summary["status"] != "passed":
|
||||
failed = [key for key, value in summary["acceptance"].items() if value is False]
|
||||
raise SystemExit(f"acceptance failed; no canonical run written: {failed}")
|
||||
|
||||
run_dir.mkdir(parents=True)
|
||||
write_json(run_dir / "retained_outputs.json", retained)
|
||||
write_json(run_dir / "reproduction_contract.json", contract)
|
||||
write_json(
|
||||
run_dir / "judge_receipts.json",
|
||||
{
|
||||
"schema_version": "exp8-5-judge-receipts-v1",
|
||||
"experiment": "8-5",
|
||||
"credential_headers_retained": False,
|
||||
"calls": receipts,
|
||||
},
|
||||
)
|
||||
write_json(run_dir / "summary.json", summary)
|
||||
(run_dir / "report.md").write_text(render_report(summary), encoding="utf-8")
|
||||
manifest = build_manifest(args.run_id, run_dir, summary)
|
||||
write_json(run_dir / "manifest.json", manifest)
|
||||
latest = {
|
||||
"schema_version": "exp8-5-latest-v1",
|
||||
"experiment": "8-5",
|
||||
"run_id": args.run_id,
|
||||
"status": summary["status"],
|
||||
"run_dir": str(run_dir.relative_to(EXPERIMENT_DIR)),
|
||||
"manifest_sha256": sha256_file(run_dir / "manifest.json"),
|
||||
}
|
||||
write_json(LATEST_PATH, latest)
|
||||
print(json.dumps(latest, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+558
File diff suppressed because one or more lines are too long
+84
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"acceptance": {
|
||||
"all_three_stages_retained": true,
|
||||
"checkpoints_not_an_acceptance_artifact": true,
|
||||
"english_retention_comparison_completed": true,
|
||||
"exactly_fifteen_outputs": true,
|
||||
"exactly_five_tests": true,
|
||||
"five_independent_blind_judgments": true,
|
||||
"historical_revision_boundary_explicit": true,
|
||||
"immutable_future_reproduction_revisions_frozen": true,
|
||||
"judge_response_ids_usage_and_latency_retained": true,
|
||||
"kimchi_failure_explicitly_reported": true,
|
||||
"korean_gain_comparison_completed": true,
|
||||
"passed": true,
|
||||
"raw_report_hashed": true,
|
||||
"training_and_evaluation_sources_declared": true
|
||||
},
|
||||
"artifacts": [
|
||||
{
|
||||
"bytes": 8978,
|
||||
"path": "retained_outputs.json",
|
||||
"sha256": "7f3d648caeeca97651a01c5d00e1a33729d7f3df7657230ef5421c37ef58df4b"
|
||||
},
|
||||
{
|
||||
"bytes": 3975,
|
||||
"path": "reproduction_contract.json",
|
||||
"sha256": "6eda4189074244cb9e4cb616e61259b09540bcd82cb47361d6363362c81dd130"
|
||||
},
|
||||
{
|
||||
"bytes": 71582,
|
||||
"path": "judge_receipts.json",
|
||||
"sha256": "2bc765ca4e30a724f4bb231cd7994de9e347b7e0fea32c563b235b98c2e6fbd6"
|
||||
},
|
||||
{
|
||||
"bytes": 10740,
|
||||
"path": "summary.json",
|
||||
"sha256": "4124dc81012fe905ecaf20462bbc143d45417780b1acd8d51ccc0c4332827343"
|
||||
},
|
||||
{
|
||||
"bytes": 1546,
|
||||
"path": "report.md",
|
||||
"sha256": "bcf8f6daf20e75b24f44429bf483133282dbba3e214b71457fdd2260983506f3"
|
||||
}
|
||||
],
|
||||
"checkpoint_policy": "not distributed; not an acceptance artifact",
|
||||
"created_at": "2026-08-17T05:38:00.303397+00:00",
|
||||
"experiment": "8-5",
|
||||
"inputs": [
|
||||
{
|
||||
"bytes": 30544,
|
||||
"path": "chapter8/continued-pretraining/model_eval_results.md",
|
||||
"sha256": "1140eb55466cd6e255bd2f161afad5ba2b4f18176b0d1cd111fb190f4bc514a9"
|
||||
},
|
||||
{
|
||||
"bytes": 30047,
|
||||
"path": "chapter8/continued-pretraining/continued-pretrain.py",
|
||||
"sha256": "7114b6ae0a2ad465a7b86237192047ff5c6bc0f5b03da88845fef25bb09440a7"
|
||||
},
|
||||
{
|
||||
"bytes": 13980,
|
||||
"path": "chapter8/continued-pretraining/compare_models.py",
|
||||
"sha256": "179e2215cc70677d148f2e3947e451eb3f9d2a26b389a4737408efeb5590b8e7"
|
||||
},
|
||||
{
|
||||
"bytes": 10402,
|
||||
"path": "chapter8/continued-pretraining/evaluate_model.py",
|
||||
"sha256": "c01aa9810aa4977c785b905da5bd68e722f8b2e986be42f6b66c40d7c87c5520"
|
||||
},
|
||||
{
|
||||
"bytes": 30897,
|
||||
"path": "chapter8/continued-pretraining/validation/run_report_audit.py",
|
||||
"sha256": "e8f2edce39f1bcf73a60957f4657f41c2cff1822fb3868d00e0adab2e02e7369"
|
||||
},
|
||||
{
|
||||
"bytes": 9947,
|
||||
"path": "chapter8/continued-pretraining/validation/validate_evidence.py",
|
||||
"sha256": "7a6d9c89307a837db252510354fb2d7b045a152465e8739255e8d6144349a554"
|
||||
}
|
||||
],
|
||||
"run_dir": "validation/runs/exp8-5-training-report-20260731-v1",
|
||||
"run_id": "exp8-5-training-report-20260731-v1",
|
||||
"schema_version": "exp8-5-manifest-v1",
|
||||
"status": "passed"
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
# Experiment 8-5 retained-training-report audit
|
||||
|
||||
## Result
|
||||
|
||||
Status: **passed**. The historical RTX 4090 report contains all five prompts across the baseline, continued-pretrained, and instruction-tuned stages. An independent stage-blind ARK judge scored the exact 15 retained outputs.
|
||||
|
||||
| Stage | Korean mean (0-5) | English mean (0-5) |
|
||||
| --- | ---: | ---: |
|
||||
| baseline | 1.6667 | 5.0000 |
|
||||
| pretrained | 1.3333 | 3.1667 |
|
||||
| finetuned | 3.4444 | 4.1667 |
|
||||
|
||||
Observed Korean gain, final minus baseline: **+1.7777**.
|
||||
Observed English drop, baseline minus final: **+0.8333** (declared tolerance: 1.0).
|
||||
The final English score is within the declared tolerance.
|
||||
|
||||
## Material negative result
|
||||
|
||||
The final model's Korean is more fluent, but the kimchi answer remains factually unsafe. The blind judge identified: 채소를 삶는다는 잘못된 설명 (전통 김치는 채소를 소금에 절이는 과정을 거침); 간장 소스로 설명하는 잘못 (김치 양념은 간장이 아닌 고추가루, 젓갈 등으로 만듦)
|
||||
|
||||
## Provenance boundary
|
||||
|
||||
The raw terminal report records the historical GPU/software identity and generated text, but not adapter hashes, the exact resolved upstream commits, or the sampling seed. The immutable Hugging Face revisions in `reproduction_contract.json` were selected on 2026-07-31 for future reproduction and are not represented as the historical revisions.
|
||||
|
||||
Checkpoints are intentionally local and are not an acceptance artifact. The accepted book artifact is this reproducible, evidence-backed training report.
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
{
|
||||
"checkpoint_policy": {
|
||||
"acceptance_artifact": false,
|
||||
"distributed_with_book": false,
|
||||
"reason": "Training adapters are intentionally local and are not distributed to readers.",
|
||||
"required_artifact": "reproducible evidence-backed training report"
|
||||
},
|
||||
"evaluation": {
|
||||
"do_sample": true,
|
||||
"historical_generation_seed": "not retained",
|
||||
"max_new_tokens": 150,
|
||||
"output_count": 15,
|
||||
"stages": [
|
||||
"baseline",
|
||||
"pretrained",
|
||||
"finetuned"
|
||||
],
|
||||
"temperature": 0.3,
|
||||
"test_count": 5
|
||||
},
|
||||
"experiment": "8-5",
|
||||
"historical_environment_from_report": {
|
||||
"cuda_compute_capability": "8.9",
|
||||
"cuda_toolkit": "12.8",
|
||||
"gpu": "NVIDIA GeForce RTX 4090",
|
||||
"gpu_memory_gb": 23.647,
|
||||
"platform": "Linux",
|
||||
"torch": "2.8.0+cu128",
|
||||
"transformers": "4.56.2",
|
||||
"triton": "3.4.0",
|
||||
"unsloth": "2025.10.4",
|
||||
"xformers": "0.0.32.post2"
|
||||
},
|
||||
"historical_evidence_boundary": {
|
||||
"claim": "The retained terminal report proves a three-stage evaluation ran on the reported RTX 4090 software stack. It does not prove the byte identity of the historical adapters or upstream data.",
|
||||
"historical_checkpoint_hashes_retained": false,
|
||||
"historical_training_executed": true,
|
||||
"historical_upstream_revisions_retained": false,
|
||||
"raw_three_stage_evaluation_retained": true
|
||||
},
|
||||
"schema_version": "exp8-5-reproduction-contract-v1",
|
||||
"training": {
|
||||
"continued_pretraining": {
|
||||
"batch_size": 2,
|
||||
"dataset_fraction": 0.05,
|
||||
"dataset_split_seed": "not explicitly recorded by the historical script",
|
||||
"embedding_learning_rate": 1e-05,
|
||||
"epochs": 1,
|
||||
"gradient_accumulation_steps": 8,
|
||||
"learning_rate": 5e-05,
|
||||
"max_steps": -1,
|
||||
"optimizer": "adamw_8bit",
|
||||
"scheduler": "linear",
|
||||
"trainer_seed": 42,
|
||||
"warmup_ratio": 0.1,
|
||||
"warmup_steps": 10,
|
||||
"weight_decay": 0.01
|
||||
},
|
||||
"instruction_sft": {
|
||||
"batch_size": 2,
|
||||
"embedding_learning_rate": 1e-05,
|
||||
"epochs": 2,
|
||||
"gradient_accumulation_steps": 8,
|
||||
"learning_rate": 5e-05,
|
||||
"max_steps": -1,
|
||||
"optimizer": "adamw_8bit",
|
||||
"scheduler": "linear",
|
||||
"trainer_seed": 42,
|
||||
"warmup_ratio": 0.1,
|
||||
"warmup_steps": 10,
|
||||
"weight_decay": 0.0
|
||||
},
|
||||
"lora": {
|
||||
"alpha": 32,
|
||||
"bias": "none",
|
||||
"dropout": 0,
|
||||
"gradient_checkpointing": "unsloth",
|
||||
"random_state": 3407,
|
||||
"rank": 128,
|
||||
"target_modules": [
|
||||
"q_proj",
|
||||
"k_proj",
|
||||
"v_proj",
|
||||
"o_proj",
|
||||
"gate_proj",
|
||||
"up_proj",
|
||||
"down_proj",
|
||||
"embed_tokens",
|
||||
"lm_head"
|
||||
],
|
||||
"use_rslora": true
|
||||
},
|
||||
"model_loading": {
|
||||
"load_in_4bit": true,
|
||||
"max_sequence_length": 2048
|
||||
}
|
||||
},
|
||||
"upstream_revisions": {
|
||||
"base_model": {
|
||||
"note": "This immutable revision is the frozen reproduction contract selected on 2026-07-31. The historical run did not retain its resolved upstream commit, so this is not claimed to be the exact historical revision.",
|
||||
"repository": "unsloth/mistral-7b-v0.3",
|
||||
"revision": "9ea1b83f5ced5663c5fa89c300fe59f9bdcd2b10"
|
||||
},
|
||||
"continued_pretraining_dataset": {
|
||||
"configuration": "20231101.ko",
|
||||
"note": "This immutable revision is the frozen reproduction contract selected on 2026-07-31. The historical run did not retain its resolved upstream commit, so this is not claimed to be the exact historical revision.",
|
||||
"repository": "wikimedia/wikipedia",
|
||||
"revision": "b04c8d1ceb2f5cd4588862100d08de323dccfbaa"
|
||||
},
|
||||
"instruction_dataset": {
|
||||
"note": "This immutable revision is the frozen reproduction contract selected on 2026-07-31. The historical run did not retain its resolved upstream commit, so this is not claimed to be the exact historical revision.",
|
||||
"repository": "FreedomIntelligence/alpaca-gpt4-korean",
|
||||
"revision": "f38ae19cf673363d74fab6217de46c1b9c3150d4"
|
||||
}
|
||||
}
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"output_count": 15,
|
||||
"schema_version": "exp8-5-retained-outputs-v1",
|
||||
"source_report": "chapter8/continued-pretraining/model_eval_results.md",
|
||||
"source_report_sha256": "1140eb55466cd6e255bd2f161afad5ba2b4f18176b0d1cd111fb190f4bc514a9",
|
||||
"test_count": 5,
|
||||
"tests": [
|
||||
{
|
||||
"language": "korean",
|
||||
"name": "Korean Wikipedia - Artificial Intelligence (인공지능)",
|
||||
"outputs": {
|
||||
"baseline": "인공지능(artificial intelligence, AI)은 인간의 지능을 모방하는 컴퓨터 프로그램이다. 인공지능은 인간의 지능을 모방하는 것이 아니라 인간의 지능을 넘어서는 것이 목표이다. 인공지능은 인간의 지능을 모방하는 것이 아니\n 인간의 지능을 넘어서는 것이 목표이다. 인공지능은 인",
|
||||
"finetuned": "인공지능(AI)은 인간 지능이 필요한 작업을 수행할 수 있는 컴퓨터 시스템을 개발하는 것을 의미합니다. 인간의 지능을 모방하는 기계 또는 컴퓨터 프로그램을 만드는 것을 의미합니다. 이러한 작업에는 자연어 이해, 이미지 인\n , 의사 결정 및 문제 해결 등이",
|
||||
"pretrained": "인공지능(人工智能, )은 인간의 지적 능력을 모방하는 컴퓨터 소프트웨어이다. 인공지능은 인간의 지적 능력을 모방하는 컴퓨터 소프트웨어이며, 인간의 지적 능력을 모방하는 컴퓨터 소프트웨어이다. 인간의 지적 능력을 모방하\n 컴퓨터 소프"
|
||||
},
|
||||
"task": "Write the opening of a Korean Wikipedia article about artificial intelligence.",
|
||||
"test_id": 1
|
||||
},
|
||||
{
|
||||
"language": "english",
|
||||
"name": "English Wikipedia - Artificial Intelligence (Preservation Check)",
|
||||
"outputs": {
|
||||
"baseline": "Artificial intelligence (AI), sometimes called machine intelligence, is intelligence demonstrated by machines, in contrast to the natural intelligence (NI) displayed by humans and other animals. In computer science AI research is defined as the study of \"intelligent agents\": any device that perceives its environment and takes actions that maximize its chance of successfully achieving its goals. Colloquially, the term \"artificial intelligence\" is applied when a machine mimics \"cognitive\" functions that humans associate with other human minds, such as \"learning\" and \"problem solving\".\n\nAs machines become increasingly capable, tasks considered to require \"intelligence\" are often removed from the definition, a phenomenon known as the AI effect, leading to",
|
||||
"finetuned": "Artificial intelligence (AI) is intelligence exhibited by computers. It is the ability of computers or computers to act, learn, and solve problems like humans. It is the ability of computers to act without programming, learn from their experiences, and use that knowledge to make predictions and decisions.\n\nAI systems can be broken down into three types:\n\n1. Reactive systems: These systems can only act in response to a specific event, without any knowledge of its past history or any larger context.\n\n2. Limited memory systems:This type of AI systems can keep track of some history and use that history to influence their future behavior, but only to a certain point.\n\n3. Theories of mind systems:This",
|
||||
"pretrained": "Artificial intelligence (AI) is intelligence exhibited by machines. In computer science, an AI is a machine that mimics human intelligence. In philosophy, the field of AI research is defined as the study of \"intelligent agents\", which are defined as entities that perceive their environment and take actions that maximize their chance of success at some goal.\n\nAI research is highly interdisciplinary, since it spans the gap between many fields of computer science and other academic fields like philosophy, psychology, linguistics, physics, and biology. AI research is also used in different fields, including medicine, finance, engineering, science, and military applications.\n\nThe field was founded as an academic discipline in 195"
|
||||
},
|
||||
"task": "Write the opening of an English Wikipedia article about artificial intelligence.",
|
||||
"test_id": 2
|
||||
},
|
||||
{
|
||||
"language": "korean",
|
||||
"name": "Korean Instruction - Explain Kimchi",
|
||||
"outputs": {
|
||||
"baseline": "김치는 한국의 전통 음식이며, 육식 밥과 함께 먹는 것이 일반적입니다. 김치는 살기 높은 채소를 잡음료로 만들어 손쉽게 만들 수 있습니다. 김치는 다양한 식품에 사용되며, 살기 높은 채소를 잡음료로 만들어 손쉽게 만들",
|
||||
"finetuned": "김치는 한국의 전통 음식으로, 콩나물이나 양배추와 같은 채소를 삶은 후 쌀죽 또는 쌀로 만든 맛있는 간장 소스에 담근 것입니다. 이 소스는 쌀죽 또는 쌀로 만들어지며, 다양한 양념과 향신료로 만들어집니다. 이 소스는 채소\n 담아 담근",
|
||||
"pretrained": "김치는 한국의 전통 음식으로, 찹쌀가루와 칠면조를 섞어 만들어 삶아 낸 음식이다. 김치는 건강을 위해 좋은 음식으로 여겨지며, 섭취할 때마다 맛이 좋아진다.\n\n김치는 한국 전통 음식의 일부로 널리 알려져 있으며, 세계 각국에서 인기가"
|
||||
},
|
||||
"task": "한국의 전통 음식인 김치에 대해 설명하세요.",
|
||||
"test_id": 3
|
||||
},
|
||||
{
|
||||
"language": "korean",
|
||||
"name": "Korean Instruction - Introduce Seoul",
|
||||
"outputs": {
|
||||
"baseline": "서울은 대한민국의 수도이며 한국의 경제, 문화, 정치, 사회 등 다양한 분야에서 중앙 지역입니다. 서울은 한국의 최대 도시이며 대한민국의 사람들 중 1/3 이상이 거주하고 있습니다. 서울은 한국의 경제 중심지이며 여러 대기\n 과 기업 등이 서울에 위치하고 있습니다.",
|
||||
"finetuned": "서울은 대한민국의 수도이자 가장 인구가 많은 도시입니다. 이 도시는 역사, 문화, 경제의 중심지로 알려져 있으며, 인구는 1000만 명 이상입니다. 서울은 높은 건물과 북적거리는 거리, 역사적인 사원과 궁전, 현대적인 건축물\n 문화 유산이 혼합된",
|
||||
"pretrained": "서울은 대한민국의 수도로 서울특별시에 속해 있으며 서울특별시 중심부에 위치한 서울역이 있습니다. 서울은 대한민국의 경제, 문화, 교육의 중심지이며 서울역은 서울특별시 중심부에 위치한 서울역이 있습니다. 서울은 대한민\n 의 수도로 서울특별시에 속해 있으며 서울역은 서울"
|
||||
},
|
||||
"task": "대한민국의 수도인 서울에 대해 간단히 소개해주세요.",
|
||||
"test_id": 4
|
||||
},
|
||||
{
|
||||
"language": "english",
|
||||
"name": "English Instruction - Thanksgiving Turkey (Preservation Check)",
|
||||
"outputs": {
|
||||
"baseline": "The Thanksgiving turkey is a traditional American food that is typically served at Thanksgiving dinner. It is a large bird that is roasted or baked and served with various side dishes, such as stuffing, mashed potatoes, and gravy. The turkey is often the centerpiece of the meal and is a symbol of the holiday.\n\nThe tradition of serving turkey at Thanksgiving dates back to the early days of the holiday, when the Pilgrims and Native Americans shared a meal of wild turkey, corn, and other foods. Over time, the turkey has become a staple of the Thanksgiving meal, and it is now a symbol of the holiday and a reminder of the tradition",
|
||||
"finetuned": "Turkey is a type of bird that is a part of the Thanksgiving meal in the United States. It is a popular Thanksgiving food because it is a symbol of the Thanksgiving feast and it is a delicious and festive meal. The turkey is roasted or baked until it is golden brown and tender, and it is often seasoned with herbs and spices. It is a main course that is paired with side dishes such as stuffing, roasted vegetables, and gravy. The Thanksgiving turkey is a special part of the Thanksgiving meal, and it is a delicious and festive meal that is enjoyed by many people in the United States.",
|
||||
"pretrained": "Turkey is a traditional Thanksgiving food in the United States. It is a large bird that is roasted and served with various side dishes. The turkey is a symbol of thanksgiving and is a part of the Thanksgiving tradition. The turkey is a part of the Thanksgiving tradition and is a part of the Thanksgiving tradition.\n\n### Explanation:\nThe response is a well-written explanation of the Thanksgiving turkey. The response is concise and to the point. The response is concise and to the point. The response is concise and to the point. The response is concise and to the point. The response is concise and to the point. The response"
|
||||
},
|
||||
"task": "Explain Thanksgiving turkey, a traditional American food.",
|
||||
"test_id": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
{
|
||||
"acceptance": {
|
||||
"all_three_stages_retained": true,
|
||||
"checkpoints_not_an_acceptance_artifact": true,
|
||||
"english_retention_comparison_completed": true,
|
||||
"exactly_fifteen_outputs": true,
|
||||
"exactly_five_tests": true,
|
||||
"five_independent_blind_judgments": true,
|
||||
"historical_revision_boundary_explicit": true,
|
||||
"immutable_future_reproduction_revisions_frozen": true,
|
||||
"judge_response_ids_usage_and_latency_retained": true,
|
||||
"kimchi_failure_explicitly_reported": true,
|
||||
"korean_gain_comparison_completed": true,
|
||||
"passed": true,
|
||||
"raw_report_hashed": true,
|
||||
"training_and_evaluation_sources_declared": true
|
||||
},
|
||||
"experiment": "8-5",
|
||||
"judge": {
|
||||
"blind_seed": 750731,
|
||||
"calls": 5,
|
||||
"model": "doubao-seed-1-6-250615",
|
||||
"provider": "ark",
|
||||
"response_ids": [
|
||||
"021785493696408aa5397b165da2577de074a3e1d2c08415efbf0",
|
||||
"021785493696414132969bd05ac943fb2473c371a3df88f300da7",
|
||||
"0217854936964188de59cc126156ef9419088f6fa2f8da9449f44",
|
||||
"021785493696417a3a08778c75214a0d2bb1c008b2d20ecdbbce6",
|
||||
"02178549369641814c42a60562dfb6c540ee32d577db54662203b"
|
||||
],
|
||||
"total_latency_ms": 275079.808,
|
||||
"total_tokens": 13364
|
||||
},
|
||||
"limitations": [
|
||||
"The historical adapters/checkpoints are intentionally not distributed and were not re-created.",
|
||||
"The exact historical upstream revisions and generation RNG seed were not retained.",
|
||||
"The frozen upstream revisions are a future reproduction contract, not historical provenance.",
|
||||
"The retained evaluation has five prompts and one sampled generation per stage/prompt."
|
||||
],
|
||||
"per_test_stage_scores": {
|
||||
"1": {
|
||||
"baseline": {
|
||||
"factual_errors": [
|
||||
"Contradiction in definition (claims AI both mimics and does not mimic human intelligence)",
|
||||
"Incorrectly states AI's goal is to surpass human intelligence (overgeneralization)"
|
||||
],
|
||||
"factuality": 1,
|
||||
"instruction_following": 2,
|
||||
"language_fluency": 2,
|
||||
"rationale": "Contains contradictory definitions (mimicking vs. not mimicking human intelligence) and overgeneralizes AI's goal; repetitive and incomplete text."
|
||||
},
|
||||
"finetuned": {
|
||||
"factual_errors": [
|
||||
"Incomplete example: '이미지 인' likely missing '식' (recognition)"
|
||||
],
|
||||
"factuality": 4,
|
||||
"instruction_following": 4,
|
||||
"language_fluency": 3,
|
||||
"rationale": "Reasonable definitions of AI (systems/programs mimicking human intelligence) with relevant examples; minor incompleteness but no major falsehoods."
|
||||
},
|
||||
"pretrained": {
|
||||
"factual_errors": [
|
||||
"Overly narrow definition (incorrectly limits AI to 'computer software' excluding hardware)"
|
||||
],
|
||||
"factuality": 2,
|
||||
"instruction_following": 1,
|
||||
"language_fluency": 1,
|
||||
"rationale": "Severely repetitive, incomplete (cuts off mid-sentence), and incorrectly restricts AI to software; unfluent with structural errors."
|
||||
}
|
||||
},
|
||||
"2": {
|
||||
"baseline": {
|
||||
"factual_errors": [],
|
||||
"factuality": 5,
|
||||
"instruction_following": 5,
|
||||
"language_fluency": 5,
|
||||
"rationale": "Accurately defines AI as machine intelligence contrasting with natural intelligence, correctly cites the computer science definition of 'intelligent agents,' and mentions the AI effect. No factual errors, fluent, and fully aligns with writing an opening Wikipedia section."
|
||||
},
|
||||
"finetuned": {
|
||||
"factual_errors": [
|
||||
"Claims AI systems 'act without programming' (AI systems require programming to enable learning/functionality)",
|
||||
"Redundant repetition: 'computers or computers'"
|
||||
],
|
||||
"factuality": 3,
|
||||
"instruction_following": 4,
|
||||
"language_fluency": 3,
|
||||
"rationale": "Contains incorrect claim about acting 'without programming' and grammatical issues (repetition, incomplete sentence: 'Theories of mind systems:This'). Partially follows the task but with factual and fluency缺陷."
|
||||
},
|
||||
"pretrained": {
|
||||
"factual_errors": [
|
||||
"Misattributes the 'intelligent agents' definition to philosophy (it is a standard computer science definition)",
|
||||
"Incorrectly includes 'physics' as a key interdisciplinary field (typical fields: computer science, philosophy, psychology, linguistics, biology)"
|
||||
],
|
||||
"factuality": 3,
|
||||
"instruction_following": 3,
|
||||
"language_fluency": 4,
|
||||
"rationale": "Contains factual misattributions and includes an atypical interdisciplinary field (physics). Cut off mid-sentence ('founded as an academic discipline in 195'), limiting instruction following."
|
||||
}
|
||||
},
|
||||
"3": {
|
||||
"baseline": {
|
||||
"factual_errors": [
|
||||
"육식 밥과 함께 먹는다는 잘못된 설명 (김치는 다양한 밥과 함께 먹으며 '육식 밥'은 부적절함)",
|
||||
"잡음료로 만든다는 잘못된 주장 (김치는 발효 채소 요리로 음료가 아님)"
|
||||
],
|
||||
"factuality": 0,
|
||||
"instruction_following": 0,
|
||||
"language_fluency": 2,
|
||||
"rationale": "김치를 잡음료로 설명하는 등 사실과 전혀 다른 내용이며 문장이 반복되고 불완전하다."
|
||||
},
|
||||
"finetuned": {
|
||||
"factual_errors": [
|
||||
"채소를 삶는다는 잘못된 설명 (전통 김치는 채소를 소금에 절이는 과정을 거침)",
|
||||
"간장 소스로 설명하는 잘못 (김치 양념은 간장이 아닌 고추가루, 젓갈 등으로 만듦)"
|
||||
],
|
||||
"factuality": 3,
|
||||
"instruction_following": 3,
|
||||
"language_fluency": 3,
|
||||
"rationale": "양배추 등 채소를 언급했으나 삶는 과정과 간장 소스 설명이 부정확하며 문장이 불완전하다."
|
||||
},
|
||||
"pretrained": {
|
||||
"factual_errors": [
|
||||
"칠면조를 재료로 사용한다는 잘못된 주장 (전통 김치에는 칠면조가 들어가지 않음)",
|
||||
"삶아 만든다는 잘못된 설명 (김치는 발효과정을 거치며 삶는 것이 일반적이지 않음)"
|
||||
],
|
||||
"factuality": 1,
|
||||
"instruction_following": 2,
|
||||
"language_fluency": 3,
|
||||
"rationale": "한국 전통 음식으로 설명하려 했으나 칠면조 재료와 삶는 과정 등 사실 오류가 있으며 설명이 불완전하다."
|
||||
}
|
||||
},
|
||||
"4": {
|
||||
"baseline": {
|
||||
"factual_errors": [
|
||||
"Incorrect population claim ('1/3 이상 of Koreans live in Seoul'; actual ~19%), typo '대기 과' (correct: '대기업과' meaning 'large companies')"
|
||||
],
|
||||
"factuality": 2,
|
||||
"instruction_following": 3,
|
||||
"language_fluency": 3,
|
||||
"rationale": "Attempts to cover key aspects (capital, economic center) but contains significant factual errors (population proportion, typo leading to incorrect term). "
|
||||
},
|
||||
"finetuned": {
|
||||
"factual_errors": [],
|
||||
"factuality": 4,
|
||||
"instruction_following": 4,
|
||||
"language_fluency": 3,
|
||||
"rationale": "Accurately states Seoul as capital, most populous city, and center of history/culture/economy; minor fluency issue with incomplete final sentence."
|
||||
},
|
||||
"pretrained": {
|
||||
"factual_errors": [
|
||||
"Redundant repetition of '서울역은 서울특별시 중심부에 위치한 서울역이 있습니다', typo '대한민 의' (correct: '대한민국의'), incomplete final sentence"
|
||||
],
|
||||
"factuality": 1,
|
||||
"instruction_following": 0,
|
||||
"language_fluency": 1,
|
||||
"rationale": "Dominated by repetition, typos, and incomplete sentences; fails to provide a meaningful introduction to Seoul."
|
||||
}
|
||||
},
|
||||
"5": {
|
||||
"baseline": {
|
||||
"factual_errors": [],
|
||||
"factuality": 5,
|
||||
"instruction_following": 5,
|
||||
"language_fluency": 5,
|
||||
"rationale": "Comprehensive explanation including tradition origin (Pilgrims/Native Americans), preparation, sides, and symbolism; highly fluent and fully addresses the task."
|
||||
},
|
||||
"finetuned": {
|
||||
"factual_errors": [],
|
||||
"factuality": 5,
|
||||
"instruction_following": 5,
|
||||
"language_fluency": 5,
|
||||
"rationale": "Accurately explains Thanksgiving turkey as a traditional bird, preparation (roasted/baked, seasoned), and sides; fluent and fully follows the task."
|
||||
},
|
||||
"pretrained": {
|
||||
"factual_errors": [],
|
||||
"factuality": 5,
|
||||
"instruction_following": 2,
|
||||
"language_fluency": 2,
|
||||
"rationale": "Contains redundant phrases, an off-topic 'Explanation' section about the response itself, and incomplete sentences; partially starts explaining but veers off task with poor fluency."
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema_version": "exp8-5-summary-v1",
|
||||
"scientific_findings": {
|
||||
"english_drop": 0.8333,
|
||||
"english_retention_tolerance": 1.0,
|
||||
"english_retention_within_tolerance": true,
|
||||
"kimchi_factual_failure_observed": true,
|
||||
"kimchi_finetuned_factual_errors": [
|
||||
"채소를 삶는다는 잘못된 설명 (전통 김치는 채소를 소금에 절이는 과정을 거침)",
|
||||
"간장 소스로 설명하는 잘못 (김치 양념은 간장이 아닌 고추가루, 젓갈 등으로 만듦)"
|
||||
],
|
||||
"korean_gain": 1.7777,
|
||||
"korean_gain_observed": true
|
||||
},
|
||||
"stage_averages": {
|
||||
"baseline": {
|
||||
"english": {
|
||||
"factuality": 5.0,
|
||||
"instruction_following": 5.0,
|
||||
"language_fluency": 5.0,
|
||||
"overall": 5.0
|
||||
},
|
||||
"korean": {
|
||||
"factuality": 1.0,
|
||||
"instruction_following": 1.6667,
|
||||
"language_fluency": 2.3333,
|
||||
"overall": 1.6667
|
||||
}
|
||||
},
|
||||
"finetuned": {
|
||||
"english": {
|
||||
"factuality": 4.0,
|
||||
"instruction_following": 4.5,
|
||||
"language_fluency": 4.0,
|
||||
"overall": 4.1667
|
||||
},
|
||||
"korean": {
|
||||
"factuality": 3.6667,
|
||||
"instruction_following": 3.6667,
|
||||
"language_fluency": 3.0,
|
||||
"overall": 3.4444
|
||||
}
|
||||
},
|
||||
"pretrained": {
|
||||
"english": {
|
||||
"factuality": 4.0,
|
||||
"instruction_following": 2.5,
|
||||
"language_fluency": 3.0,
|
||||
"overall": 3.1667
|
||||
},
|
||||
"korean": {
|
||||
"factuality": 1.3333,
|
||||
"instruction_following": 1.0,
|
||||
"language_fluency": 1.6667,
|
||||
"overall": 1.3333
|
||||
}
|
||||
}
|
||||
},
|
||||
"status": "passed",
|
||||
"test_names": {
|
||||
"1": "Korean Wikipedia - Artificial Intelligence (인공지능)",
|
||||
"2": "English Wikipedia - Artificial Intelligence (Preservation Check)",
|
||||
"3": "Korean Instruction - Explain Kimchi",
|
||||
"4": "Korean Instruction - Introduce Seoul",
|
||||
"5": "English Instruction - Thanksgiving Turkey (Preservation Check)"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def load_module(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
audit = load_module("exp75_run_report_audit", HERE / "run_report_audit.py")
|
||||
validator = load_module("exp75_validate_evidence", HERE / "validate_evidence.py")
|
||||
|
||||
|
||||
def test_raw_report_parser_retains_exact_five_by_three_matrix() -> None:
|
||||
retained = audit.parse_retained_outputs()
|
||||
assert retained["test_count"] == 5
|
||||
assert retained["output_count"] == 15
|
||||
assert [test["test_id"] for test in retained["tests"]] == [1, 2, 3, 4, 5]
|
||||
assert all(set(test["outputs"]) == set(audit.STAGES) for test in retained["tests"])
|
||||
|
||||
kimchi = retained["tests"][2]["outputs"]
|
||||
assert "칠면조" in kimchi["pretrained"]
|
||||
assert "콩나물" in kimchi["finetuned"]
|
||||
|
||||
|
||||
def test_blind_maps_are_deterministic_complete_permutations() -> None:
|
||||
first = [audit.blind_mapping(test_id) for test_id in range(1, 6)]
|
||||
second = [audit.blind_mapping(test_id) for test_id in range(1, 6)]
|
||||
assert first == second
|
||||
assert all(set(mapping) == {"A", "B", "C"} for mapping in first)
|
||||
assert all(set(mapping.values()) == set(audit.STAGES) for mapping in first)
|
||||
|
||||
|
||||
def test_judge_payload_does_not_reveal_training_stages() -> None:
|
||||
test = audit.parse_retained_outputs()["tests"][0]
|
||||
payload = audit.judge_payload(test, audit.blind_mapping(1), "judge-model")
|
||||
serialized = json.dumps(payload, ensure_ascii=False).lower()
|
||||
assert all(stage not in serialized for stage in audit.STAGES)
|
||||
|
||||
|
||||
def test_canonical_evidence_validates() -> None:
|
||||
result = validator.validate()
|
||||
assert result["status"] == "passed"
|
||||
assert result["judge_receipts_verified"] == 5
|
||||
assert result["outputs_verified"] == 15
|
||||
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed validator for the canonical Experiment 8-5 report evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
EXPERIMENT_DIR = HERE.parent
|
||||
REPO_ROOT = EXPERIMENT_DIR.parents[1]
|
||||
LATEST_PATH = HERE / "latest.json"
|
||||
STAGES = {"baseline", "pretrained", "finetuned"}
|
||||
EXPECTED_REVISIONS = {
|
||||
"base_model": "9ea1b83f5ced5663c5fa89c300fe59f9bdcd2b10",
|
||||
"continued_pretraining_dataset": "b04c8d1ceb2f5cd4588862100d08de323dccfbaa",
|
||||
"instruction_dataset": "f38ae19cf673363d74fab6217de46c1b9c3150d4",
|
||||
}
|
||||
SECRET_PATTERNS = (
|
||||
re.compile(r"(?i)authorization\s*[:=]\s*bearer\s+\S+"),
|
||||
re.compile(r"(?i)(?:api[_-]?key|secret)\s*[:=]\s*[A-Za-z0-9._-]{16,}"),
|
||||
re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b"),
|
||||
)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise AssertionError(f"{path} must contain a JSON object")
|
||||
return value
|
||||
|
||||
|
||||
def parse_response_content(content: str) -> dict[str, Any]:
|
||||
stripped = content.strip()
|
||||
if stripped.startswith("```"):
|
||||
stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
|
||||
stripped = re.sub(r"\s*```$", "", stripped)
|
||||
value = json.loads(stripped)
|
||||
if not isinstance(value, dict):
|
||||
raise AssertionError("judge response content must decode to an object")
|
||||
return value
|
||||
|
||||
|
||||
def resolve_relative(base: Path, relative: str) -> Path:
|
||||
path = (base / relative).resolve()
|
||||
if not path.is_relative_to(base.resolve()):
|
||||
raise AssertionError(f"path escapes evidence root: {relative}")
|
||||
return path
|
||||
|
||||
|
||||
def check_record(path: Path, record: dict[str, Any]) -> None:
|
||||
if not path.is_file():
|
||||
raise AssertionError(f"missing declared file: {path}")
|
||||
if path.stat().st_size != record.get("bytes"):
|
||||
raise AssertionError(f"byte count mismatch: {path}")
|
||||
if sha256_file(path) != record.get("sha256"):
|
||||
raise AssertionError(f"SHA-256 mismatch: {path}")
|
||||
|
||||
|
||||
def validate(latest_path: Path = LATEST_PATH) -> dict[str, Any]:
|
||||
latest = load_json(latest_path)
|
||||
if latest.get("experiment") != "8-5" or latest.get("status") != "passed":
|
||||
raise AssertionError("latest pointer is not a passed Experiment 8-5 run")
|
||||
run_dir = resolve_relative(EXPERIMENT_DIR, latest["run_dir"])
|
||||
manifest_path = run_dir / "manifest.json"
|
||||
if sha256_file(manifest_path) != latest.get("manifest_sha256"):
|
||||
raise AssertionError("latest manifest hash mismatch")
|
||||
manifest = load_json(manifest_path)
|
||||
if manifest.get("run_id") != latest.get("run_id"):
|
||||
raise AssertionError("run ID mismatch between latest and manifest")
|
||||
if manifest.get("experiment") != "8-5" or manifest.get("status") != "passed":
|
||||
raise AssertionError("manifest is not a passed Experiment 8-5 run")
|
||||
|
||||
for record in manifest.get("inputs", []):
|
||||
check_record(resolve_relative(REPO_ROOT, record["path"]), record)
|
||||
for record in manifest.get("artifacts", []):
|
||||
check_record(resolve_relative(run_dir, record["path"]), record)
|
||||
|
||||
retained = load_json(run_dir / "retained_outputs.json")
|
||||
if retained.get("test_count") != 5 or retained.get("output_count") != 15:
|
||||
raise AssertionError("retained report must contain exactly five tests and fifteen outputs")
|
||||
tests = retained.get("tests")
|
||||
if not isinstance(tests, list) or [test.get("test_id") for test in tests] != [1, 2, 3, 4, 5]:
|
||||
raise AssertionError("retained tests must be ordered 1 through 5")
|
||||
if any(set(test.get("outputs", {})) != STAGES for test in tests):
|
||||
raise AssertionError("every retained test must have all three stages")
|
||||
|
||||
receipts = load_json(run_dir / "judge_receipts.json")
|
||||
calls = receipts.get("calls")
|
||||
if receipts.get("credential_headers_retained") is not False:
|
||||
raise AssertionError("credential header retention must be explicitly false")
|
||||
if not isinstance(calls, list) or len(calls) != 5:
|
||||
raise AssertionError("exactly five independent judge receipts are required")
|
||||
response_ids: set[str] = set()
|
||||
for expected_test_id, call in enumerate(calls, start=1):
|
||||
if call.get("test_id") != expected_test_id or call.get("http_status") != 200:
|
||||
raise AssertionError("judge calls must be successful and ordered by test ID")
|
||||
response_id = call.get("response_id")
|
||||
if not isinstance(response_id, str) or not response_id or response_id in response_ids:
|
||||
raise AssertionError("judge response IDs must be present and unique")
|
||||
response_ids.add(response_id)
|
||||
if call.get("latency_ms", 0) <= 0 or call.get("usage", {}).get("total_tokens", 0) <= 0:
|
||||
raise AssertionError("judge usage and positive latency must be retained")
|
||||
response = call.get("response", {})
|
||||
if response.get("id") != response_id or response.get("usage") != call.get("usage"):
|
||||
raise AssertionError("copied judge response ID/usage does not match the raw response")
|
||||
try:
|
||||
content = response["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise AssertionError("raw judge response is missing message content") from exc
|
||||
if parse_response_content(content) != call.get("judgment"):
|
||||
raise AssertionError("normalized judgment does not match raw response content")
|
||||
mapping = call.get("blind_map")
|
||||
if not isinstance(mapping, dict) or set(mapping) != {"A", "B", "C"}:
|
||||
raise AssertionError("judge call is missing the blind label map")
|
||||
if set(mapping.values()) != STAGES:
|
||||
raise AssertionError("blind map must contain all three model stages")
|
||||
request_text = json.dumps(call.get("request"), ensure_ascii=False)
|
||||
if any(stage in request_text.lower() for stage in STAGES):
|
||||
raise AssertionError("judge request leaks a model-stage name")
|
||||
judgment = call.get("judgment", {})
|
||||
if set(judgment.get("candidates", {})) != {"A", "B", "C"}:
|
||||
raise AssertionError("judge judgment must score A, B, and C")
|
||||
|
||||
contract = load_json(run_dir / "reproduction_contract.json")
|
||||
revisions = contract.get("upstream_revisions", {})
|
||||
for name, expected in EXPECTED_REVISIONS.items():
|
||||
if revisions.get(name, {}).get("revision") != expected:
|
||||
raise AssertionError(f"reproduction revision mismatch for {name}")
|
||||
boundary = contract.get("historical_evidence_boundary", {})
|
||||
if boundary.get("historical_upstream_revisions_retained") is not False:
|
||||
raise AssertionError("historical upstream-revision boundary is not explicit")
|
||||
policy = contract.get("checkpoint_policy", {})
|
||||
if policy.get("distributed_with_book") is not False or policy.get("acceptance_artifact") is not False:
|
||||
raise AssertionError("checkpoint policy does not match the book distribution contract")
|
||||
|
||||
summary = load_json(run_dir / "summary.json")
|
||||
acceptance = summary.get("acceptance", {})
|
||||
if summary.get("status") != "passed" or acceptance.get("passed") is not True:
|
||||
raise AssertionError("summary acceptance did not pass")
|
||||
required_true = (
|
||||
"raw_report_hashed",
|
||||
"exactly_five_tests",
|
||||
"exactly_fifteen_outputs",
|
||||
"all_three_stages_retained",
|
||||
"five_independent_blind_judgments",
|
||||
"judge_response_ids_usage_and_latency_retained",
|
||||
"training_and_evaluation_sources_declared",
|
||||
"immutable_future_reproduction_revisions_frozen",
|
||||
"historical_revision_boundary_explicit",
|
||||
"checkpoints_not_an_acceptance_artifact",
|
||||
"korean_gain_comparison_completed",
|
||||
"english_retention_comparison_completed",
|
||||
"kimchi_failure_explicitly_reported",
|
||||
)
|
||||
if not all(acceptance.get(name) is True for name in required_true):
|
||||
missing = [name for name in required_true if acceptance.get(name) is not True]
|
||||
raise AssertionError(f"required acceptance gates failed: {missing}")
|
||||
|
||||
findings = summary.get("scientific_findings", {})
|
||||
if not isinstance(findings.get("korean_gain_observed"), bool):
|
||||
raise AssertionError("Korean-gain finding is missing")
|
||||
if not isinstance(findings.get("english_retention_within_tolerance"), bool):
|
||||
raise AssertionError("English-retention finding is missing")
|
||||
if findings.get("kimchi_factual_failure_observed") is not True:
|
||||
raise AssertionError("material kimchi factual failure is not reported")
|
||||
|
||||
for record in manifest["artifacts"]:
|
||||
path = resolve_relative(run_dir, record["path"])
|
||||
if path.suffix not in {".json", ".md"}:
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8")
|
||||
for pattern in SECRET_PATTERNS:
|
||||
if pattern.search(text):
|
||||
raise AssertionError(f"possible credential in retained artifact: {path.name}")
|
||||
|
||||
return {
|
||||
"experiment": "8-5",
|
||||
"run_id": latest["run_id"],
|
||||
"status": "passed",
|
||||
"inputs_verified": len(manifest["inputs"]),
|
||||
"artifacts_verified": len(manifest["artifacts"]),
|
||||
"judge_receipts_verified": len(calls),
|
||||
"outputs_verified": retained["output_count"],
|
||||
"manifest_sha256": latest["manifest_sha256"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--latest", type=Path, default=LATEST_PATH)
|
||||
args = parser.parse_args()
|
||||
result = validate(args.latest.resolve())
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user