ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter8/continued-pretraining/validation/runs/exp8-5-training-report-20260731-v1/manifest.json

@@ -0,0 +1,84 @@
+{
+  "acceptance": {
+    "all_three_stages_retained": true,
+    "checkpoints_not_an_acceptance_artifact": true,
+    "english_retention_comparison_completed": true,
+    "exactly_fifteen_outputs": true,
+    "exactly_five_tests": true,
+    "five_independent_blind_judgments": true,
+    "historical_revision_boundary_explicit": true,
+    "immutable_future_reproduction_revisions_frozen": true,
+    "judge_response_ids_usage_and_latency_retained": true,
+    "kimchi_failure_explicitly_reported": true,
+    "korean_gain_comparison_completed": true,
+    "passed": true,
+    "raw_report_hashed": true,
+    "training_and_evaluation_sources_declared": true
+  },
+  "artifacts": [
+    {
+      "bytes": 8978,
+      "path": "retained_outputs.json",
+      "sha256": "7f3d648caeeca97651a01c5d00e1a33729d7f3df7657230ef5421c37ef58df4b"
+    },
+    {
+      "bytes": 3975,
+      "path": "reproduction_contract.json",
+      "sha256": "6eda4189074244cb9e4cb616e61259b09540bcd82cb47361d6363362c81dd130"
+    },
+    {
+      "bytes": 71582,
+      "path": "judge_receipts.json",
+      "sha256": "2bc765ca4e30a724f4bb231cd7994de9e347b7e0fea32c563b235b98c2e6fbd6"
+    },
+    {
+      "bytes": 10740,
+      "path": "summary.json",
+      "sha256": "4124dc81012fe905ecaf20462bbc143d45417780b1acd8d51ccc0c4332827343"
+    },
+    {
+      "bytes": 1546,
+      "path": "report.md",
+      "sha256": "bcf8f6daf20e75b24f44429bf483133282dbba3e214b71457fdd2260983506f3"
+    }
+  ],
+  "checkpoint_policy": "not distributed; not an acceptance artifact",
+  "created_at": "2026-08-17T05:38:00.303397+00:00",
+  "experiment": "8-5",
+  "inputs": [
+    {
+      "bytes": 30544,
+      "path": "chapter8/continued-pretraining/model_eval_results.md",
+      "sha256": "1140eb55466cd6e255bd2f161afad5ba2b4f18176b0d1cd111fb190f4bc514a9"
+    },
+    {
+      "bytes": 30047,
+      "path": "chapter8/continued-pretraining/continued-pretrain.py",
+      "sha256": "7114b6ae0a2ad465a7b86237192047ff5c6bc0f5b03da88845fef25bb09440a7"
+    },
+    {
+      "bytes": 13980,
+      "path": "chapter8/continued-pretraining/compare_models.py",
+      "sha256": "179e2215cc70677d148f2e3947e451eb3f9d2a26b389a4737408efeb5590b8e7"
+    },
+    {
+      "bytes": 10402,
+      "path": "chapter8/continued-pretraining/evaluate_model.py",
+      "sha256": "c01aa9810aa4977c785b905da5bd68e722f8b2e986be42f6b66c40d7c87c5520"
+    },
+    {
+      "bytes": 30897,
+      "path": "chapter8/continued-pretraining/validation/run_report_audit.py",
+      "sha256": "e8f2edce39f1bcf73a60957f4657f41c2cff1822fb3868d00e0adab2e02e7369"
+    },
+    {
+      "bytes": 9947,
+      "path": "chapter8/continued-pretraining/validation/validate_evidence.py",
+      "sha256": "7a6d9c89307a837db252510354fb2d7b045a152465e8739255e8d6144349a554"
+    }
+  ],
+  "run_dir": "validation/runs/exp8-5-training-report-20260731-v1",
+  "run_id": "exp8-5-training-report-20260731-v1",
+  "schema_version": "exp8-5-manifest-v1",
+  "status": "passed"
+}

chapter8/continued-pretraining/validation/runs/exp8-5-training-report-20260731-v1/report.md

@@ -0,0 +1,25 @@
+# Experiment 8-5 retained-training-report audit
+
+## Result
+
+Status: **passed**. The historical RTX 4090 report contains all five prompts across the baseline, continued-pretrained, and instruction-tuned stages. An independent stage-blind ARK judge scored the exact 15 retained outputs.
+
+| Stage | Korean mean (0-5) | English mean (0-5) |
+| --- | ---: | ---: |
+| baseline | 1.6667 | 5.0000 |
+| pretrained | 1.3333 | 3.1667 |
+| finetuned | 3.4444 | 4.1667 |
+
+Observed Korean gain, final minus baseline: **+1.7777**.
+Observed English drop, baseline minus final: **+0.8333** (declared tolerance: 1.0).
+The final English score is within the declared tolerance.
+
+## Material negative result
+
+The final model's Korean is more fluent, but the kimchi answer remains factually unsafe. The blind judge identified: 채소를 삶는다는 잘못된 설명 (전통 김치는 채소를 소금에 절이는 과정을 거침); 간장 소스로 설명하는 잘못 (김치 양념은 간장이 아닌 고추가루, 젓갈 등으로 만듦)
+
+## Provenance boundary
+
+The raw terminal report records the historical GPU/software identity and generated text, but not adapter hashes, the exact resolved upstream commits, or the sampling seed. The immutable Hugging Face revisions in `reproduction_contract.json` were selected on 2026-07-31 for future reproduction and are not represented as the historical revisions.
+
+Checkpoints are intentionally local and are not an acceptance artifact. The accepted book artifact is this reproducible, evidence-backed training report.

chapter8/continued-pretraining/validation/runs/exp8-5-training-report-20260731-v1/reproduction_contract.json

@@ -0,0 +1,116 @@
+{
+  "checkpoint_policy": {
+    "acceptance_artifact": false,
+    "distributed_with_book": false,
+    "reason": "Training adapters are intentionally local and are not distributed to readers.",
+    "required_artifact": "reproducible evidence-backed training report"
+  },
+  "evaluation": {
+    "do_sample": true,
+    "historical_generation_seed": "not retained",
+    "max_new_tokens": 150,
+    "output_count": 15,
+    "stages": [
+      "baseline",
+      "pretrained",
+      "finetuned"
+    ],
+    "temperature": 0.3,
+    "test_count": 5
+  },
+  "experiment": "8-5",
+  "historical_environment_from_report": {
+    "cuda_compute_capability": "8.9",
+    "cuda_toolkit": "12.8",
+    "gpu": "NVIDIA GeForce RTX 4090",
+    "gpu_memory_gb": 23.647,
+    "platform": "Linux",
+    "torch": "2.8.0+cu128",
+    "transformers": "4.56.2",
+    "triton": "3.4.0",
+    "unsloth": "2025.10.4",
+    "xformers": "0.0.32.post2"
+  },
+  "historical_evidence_boundary": {
+    "claim": "The retained terminal report proves a three-stage evaluation ran on the reported RTX 4090 software stack. It does not prove the byte identity of the historical adapters or upstream data.",
+    "historical_checkpoint_hashes_retained": false,
+    "historical_training_executed": true,
+    "historical_upstream_revisions_retained": false,
+    "raw_three_stage_evaluation_retained": true
+  },
+  "schema_version": "exp8-5-reproduction-contract-v1",
+  "training": {
+    "continued_pretraining": {
+      "batch_size": 2,
+      "dataset_fraction": 0.05,
+      "dataset_split_seed": "not explicitly recorded by the historical script",
+      "embedding_learning_rate": 1e-05,
+      "epochs": 1,
+      "gradient_accumulation_steps": 8,
+      "learning_rate": 5e-05,
+      "max_steps": -1,
+      "optimizer": "adamw_8bit",
+      "scheduler": "linear",
+      "trainer_seed": 42,
+      "warmup_ratio": 0.1,
+      "warmup_steps": 10,
+      "weight_decay": 0.01
+    },
+    "instruction_sft": {
+      "batch_size": 2,
+      "embedding_learning_rate": 1e-05,
+      "epochs": 2,
+      "gradient_accumulation_steps": 8,
+      "learning_rate": 5e-05,
+      "max_steps": -1,
+      "optimizer": "adamw_8bit",
+      "scheduler": "linear",
+      "trainer_seed": 42,
+      "warmup_ratio": 0.1,
+      "warmup_steps": 10,
+      "weight_decay": 0.0
+    },
+    "lora": {
+      "alpha": 32,
+      "bias": "none",
+      "dropout": 0,
+      "gradient_checkpointing": "unsloth",
+      "random_state": 3407,
+      "rank": 128,
+      "target_modules": [
+        "q_proj",
+        "k_proj",
+        "v_proj",
+        "o_proj",
+        "gate_proj",
+        "up_proj",
+        "down_proj",
+        "embed_tokens",
+        "lm_head"
+      ],
+      "use_rslora": true
+    },
+    "model_loading": {
+      "load_in_4bit": true,
+      "max_sequence_length": 2048
+    }
+  },
+  "upstream_revisions": {
+    "base_model": {
+      "note": "This immutable revision is the frozen reproduction contract selected on 2026-07-31. The historical run did not retain its resolved upstream commit, so this is not claimed to be the exact historical revision.",
+      "repository": "unsloth/mistral-7b-v0.3",
+      "revision": "9ea1b83f5ced5663c5fa89c300fe59f9bdcd2b10"
+    },
+    "continued_pretraining_dataset": {
+      "configuration": "20231101.ko",
+      "note": "This immutable revision is the frozen reproduction contract selected on 2026-07-31. The historical run did not retain its resolved upstream commit, so this is not claimed to be the exact historical revision.",
+      "repository": "wikimedia/wikipedia",
+      "revision": "b04c8d1ceb2f5cd4588862100d08de323dccfbaa"
+    },
+    "instruction_dataset": {
+      "note": "This immutable revision is the frozen reproduction contract selected on 2026-07-31. The historical run did not retain its resolved upstream commit, so this is not claimed to be the exact historical revision.",
+      "repository": "FreedomIntelligence/alpaca-gpt4-korean",
+      "revision": "f38ae19cf673363d74fab6217de46c1b9c3150d4"
+    }
+  }
+}

chapter8/continued-pretraining/validation/runs/exp8-5-training-report-20260731-v1/retained_outputs.json

@@ -0,0 +1,64 @@
+{
+  "output_count": 15,
+  "schema_version": "exp8-5-retained-outputs-v1",
+  "source_report": "chapter8/continued-pretraining/model_eval_results.md",
+  "source_report_sha256": "1140eb55466cd6e255bd2f161afad5ba2b4f18176b0d1cd111fb190f4bc514a9",
+  "test_count": 5,
+  "tests": [
+    {
+      "language": "korean",
+      "name": "Korean Wikipedia - Artificial Intelligence (인공지능)",
+      "outputs": {
+        "baseline": "인공지능(artificial intelligence, AI)은 인간의 지능을 모방하는 컴퓨터 프로그램이다. 인공지능은 인간의 지능을 모방하는 것이 아니라 인간의 지능을 넘어서는 것이 목표이다. 인공지능은 인간의 지능을 모방하는 것이 아니\n   인간의 지능을 넘어서는 것이 목표이다. 인공지능은 인",
+        "finetuned": "인공지능(AI)은 인간 지능이 필요한 작업을 수행할 수 있는 컴퓨터 시스템을 개발하는 것을 의미합니다. 인간의 지능을 모방하는 기계 또는 컴퓨터 프로그램을 만드는 것을 의미합니다. 이러한 작업에는 자연어 이해, 이미지 인\n  , 의사 결정 및 문제 해결 등이",
+        "pretrained": "인공지능(人工智能, )은 인간의 지적 능력을 모방하는 컴퓨터 소프트웨어이다. 인공지능은 인간의 지적 능력을 모방하는 컴퓨터 소프트웨어이며, 인간의 지적 능력을 모방하는 컴퓨터 소프트웨어이다. 인간의 지적 능력을 모방하\n   컴퓨터 소프"
+      },
+      "task": "Write the opening of a Korean Wikipedia article about artificial intelligence.",
+      "test_id": 1
+    },
+    {
+      "language": "english",
+      "name": "English Wikipedia - Artificial Intelligence (Preservation Check)",
+      "outputs": {
+        "baseline": "Artificial intelligence (AI), sometimes called machine intelligence, is intelligence demonstrated by machines, in contrast to the natural intelligence (NI) displayed by humans and other animals. In computer science AI research is defined as the study of \"intelligent agents\": any device that perceives its environment and takes actions that maximize its chance of successfully achieving its goals. Colloquially, the term \"artificial intelligence\" is applied when a machine mimics \"cognitive\" functions that humans associate with other human minds, such as \"learning\" and \"problem solving\".\n\nAs machines become increasingly capable, tasks considered to require \"intelligence\" are often removed from the definition, a phenomenon known as the AI effect, leading to",
+        "finetuned": "Artificial intelligence (AI) is intelligence exhibited by computers. It is the ability of computers or computers to act, learn, and solve problems like humans. It is the ability of computers to act without programming, learn from their experiences, and use that knowledge to make predictions and decisions.\n\nAI systems can be broken down into three types:\n\n1. Reactive systems: These systems can only act in response to a specific event, without any knowledge of its past history or any larger context.\n\n2. Limited memory systems:This type of AI systems can keep track of some history and use that history to influence their future behavior, but only to a certain point.\n\n3. Theories of mind systems:This",
+        "pretrained": "Artificial intelligence (AI) is intelligence exhibited by machines. In computer science, an AI is a machine that mimics human intelligence. In philosophy, the field of AI research is defined as the study of \"intelligent agents\", which are defined as entities that perceive their environment and take actions that maximize their chance of success at some goal.\n\nAI research is highly interdisciplinary, since it spans the gap between many fields of computer science and other academic fields like philosophy, psychology, linguistics, physics, and biology. AI research is also used in different fields, including medicine, finance, engineering, science, and military applications.\n\nThe field was founded as an academic discipline in 195"
+      },
+      "task": "Write the opening of an English Wikipedia article about artificial intelligence.",
+      "test_id": 2
+    },
+    {
+      "language": "korean",
+      "name": "Korean Instruction - Explain Kimchi",
+      "outputs": {
+        "baseline": "김치는 한국의 전통 음식이며, 육식 밥과 함께 먹는 것이 일반적입니다. 김치는 살기 높은 채소를 잡음료로 만들어 손쉽게 만들 수 있습니다. 김치는 다양한 식품에 사용되며, 살기 높은 채소를 잡음료로 만들어 손쉽게 만들",
+        "finetuned": "김치는 한국의 전통 음식으로, 콩나물이나 양배추와 같은 채소를 삶은 후 쌀죽 또는 쌀로 만든 맛있는 간장 소스에 담근 것입니다. 이 소스는 쌀죽 또는 쌀로 만들어지며, 다양한 양념과 향신료로 만들어집니다. 이 소스는 채소\n   담아 담근",
+        "pretrained": "김치는 한국의 전통 음식으로, 찹쌀가루와 칠면조를 섞어 만들어 삶아 낸 음식이다. 김치는 건강을 위해 좋은 음식으로 여겨지며, 섭취할 때마다 맛이 좋아진다.\n\n김치는 한국 전통 음식의 일부로 널리 알려져 있으며, 세계 각국에서 인기가"
+      },
+      "task": "한국의 전통 음식인 김치에 대해 설명하세요.",
+      "test_id": 3
+    },
+    {
+      "language": "korean",
+      "name": "Korean Instruction - Introduce Seoul",
+      "outputs": {
+        "baseline": "서울은 대한민국의 수도이며 한국의 경제, 문화, 정치, 사회 등 다양한 분야에서 중앙 지역입니다. 서울은 한국의 최대 도시이며 대한민국의 사람들 중 1/3 이상이 거주하고 있습니다. 서울은 한국의 경제 중심지이며 여러 대기\n  과 기업 등이 서울에 위치하고 있습니다.",
+        "finetuned": "서울은 대한민국의 수도이자 가장 인구가 많은 도시입니다. 이 도시는 역사, 문화, 경제의 중심지로 알려져 있으며, 인구는 1000만 명 이상입니다. 서울은 높은 건물과 북적거리는 거리, 역사적인 사원과 궁전, 현대적인 건축물\n   문화 유산이 혼합된",
+        "pretrained": "서울은 대한민국의 수도로 서울특별시에 속해 있으며 서울특별시 중심부에 위치한 서울역이 있습니다. 서울은 대한민국의 경제, 문화, 교육의 중심지이며 서울역은 서울특별시 중심부에 위치한 서울역이 있습니다. 서울은 대한민\n  의 수도로 서울특별시에 속해 있으며 서울역은 서울"
+      },
+      "task": "대한민국의 수도인 서울에 대해 간단히 소개해주세요.",
+      "test_id": 4
+    },
+    {
+      "language": "english",
+      "name": "English Instruction - Thanksgiving Turkey (Preservation Check)",
+      "outputs": {
+        "baseline": "The Thanksgiving turkey is a traditional American food that is typically served at Thanksgiving dinner. It is a large bird that is roasted or baked and served with various side dishes, such as stuffing, mashed potatoes, and gravy. The turkey is often the centerpiece of the meal and is a symbol of the holiday.\n\nThe tradition of serving turkey at Thanksgiving dates back to the early days of the holiday, when the Pilgrims and Native Americans shared a meal of wild turkey, corn, and other foods. Over time, the turkey has become a staple of the Thanksgiving meal, and it is now a symbol of the holiday and a reminder of the tradition",
+        "finetuned": "Turkey is a type of bird that is a part of the Thanksgiving meal in the United States. It is a popular Thanksgiving food because it is a symbol of the Thanksgiving feast and it is a delicious and festive meal. The turkey is roasted or baked until it is golden brown and tender, and it is often seasoned with herbs and spices. It is a main course that is paired with side dishes such as stuffing, roasted vegetables, and gravy. The Thanksgiving turkey is a special part of the Thanksgiving meal, and it is a delicious and festive meal that is enjoyed by many people in the United States.",
+        "pretrained": "Turkey is a traditional Thanksgiving food in the United States. It is a large bird that is roasted and served with various side dishes. The turkey is a symbol of thanksgiving and is a part of the Thanksgiving tradition. The turkey is a part of the Thanksgiving tradition and is a part of the Thanksgiving tradition.\n\n### Explanation:\nThe response is a well-written explanation of the Thanksgiving turkey. The response is concise and to the point. The response is concise and to the point. The response is concise and to the point. The response is concise and to the point. The response is concise and to the point. The response"
+      },
+      "task": "Explain Thanksgiving turkey, a traditional American food.",
+      "test_id": 5
+    }
+  ]
+}

chapter8/continued-pretraining/validation/runs/exp8-5-training-report-20260731-v1/summary.json

@@ -0,0 +1,248 @@
+{
+  "acceptance": {
+    "all_three_stages_retained": true,
+    "checkpoints_not_an_acceptance_artifact": true,
+    "english_retention_comparison_completed": true,
+    "exactly_fifteen_outputs": true,
+    "exactly_five_tests": true,
+    "five_independent_blind_judgments": true,
+    "historical_revision_boundary_explicit": true,
+    "immutable_future_reproduction_revisions_frozen": true,
+    "judge_response_ids_usage_and_latency_retained": true,
+    "kimchi_failure_explicitly_reported": true,
+    "korean_gain_comparison_completed": true,
+    "passed": true,
+    "raw_report_hashed": true,
+    "training_and_evaluation_sources_declared": true
+  },
+  "experiment": "8-5",
+  "judge": {
+    "blind_seed": 750731,
+    "calls": 5,
+    "model": "doubao-seed-1-6-250615",
+    "provider": "ark",
+    "response_ids": [
+      "021785493696408aa5397b165da2577de074a3e1d2c08415efbf0",
+      "021785493696414132969bd05ac943fb2473c371a3df88f300da7",
+      "0217854936964188de59cc126156ef9419088f6fa2f8da9449f44",
+      "021785493696417a3a08778c75214a0d2bb1c008b2d20ecdbbce6",
+      "02178549369641814c42a60562dfb6c540ee32d577db54662203b"
+    ],
+    "total_latency_ms": 275079.808,
+    "total_tokens": 13364
+  },
+  "limitations": [
+    "The historical adapters/checkpoints are intentionally not distributed and were not re-created.",
+    "The exact historical upstream revisions and generation RNG seed were not retained.",
+    "The frozen upstream revisions are a future reproduction contract, not historical provenance.",
+    "The retained evaluation has five prompts and one sampled generation per stage/prompt."
+  ],
+  "per_test_stage_scores": {
+    "1": {
+      "baseline": {
+        "factual_errors": [
+          "Contradiction in definition (claims AI both mimics and does not mimic human intelligence)",
+          "Incorrectly states AI's goal is to surpass human intelligence (overgeneralization)"
+        ],
+        "factuality": 1,
+        "instruction_following": 2,
+        "language_fluency": 2,
+        "rationale": "Contains contradictory definitions (mimicking vs. not mimicking human intelligence) and overgeneralizes AI's goal; repetitive and incomplete text."
+      },
+      "finetuned": {
+        "factual_errors": [
+          "Incomplete example: '이미지 인' likely missing '식' (recognition)"
+        ],
+        "factuality": 4,
+        "instruction_following": 4,
+        "language_fluency": 3,
+        "rationale": "Reasonable definitions of AI (systems/programs mimicking human intelligence) with relevant examples; minor incompleteness but no major falsehoods."
+      },
+      "pretrained": {
+        "factual_errors": [
+          "Overly narrow definition (incorrectly limits AI to 'computer software' excluding hardware)"
+        ],
+        "factuality": 2,
+        "instruction_following": 1,
+        "language_fluency": 1,
+        "rationale": "Severely repetitive, incomplete (cuts off mid-sentence), and incorrectly restricts AI to software; unfluent with structural errors."
+      }
+    },
+    "2": {
+      "baseline": {
+        "factual_errors": [],
+        "factuality": 5,
+        "instruction_following": 5,
+        "language_fluency": 5,
+        "rationale": "Accurately defines AI as machine intelligence contrasting with natural intelligence, correctly cites the computer science definition of 'intelligent agents,' and mentions the AI effect. No factual errors, fluent, and fully aligns with writing an opening Wikipedia section."
+      },
+      "finetuned": {
+        "factual_errors": [
+          "Claims AI systems 'act without programming' (AI systems require programming to enable learning/functionality)",
+          "Redundant repetition: 'computers or computers'"
+        ],
+        "factuality": 3,
+        "instruction_following": 4,
+        "language_fluency": 3,
+        "rationale": "Contains incorrect claim about acting 'without programming' and grammatical issues (repetition, incomplete sentence: 'Theories of mind systems:This'). Partially follows the task but with factual and fluency缺陷."
+      },
+      "pretrained": {
+        "factual_errors": [
+          "Misattributes the 'intelligent agents' definition to philosophy (it is a standard computer science definition)",
+          "Incorrectly includes 'physics' as a key interdisciplinary field (typical fields: computer science, philosophy, psychology, linguistics, biology)"
+        ],
+        "factuality": 3,
+        "instruction_following": 3,
+        "language_fluency": 4,
+        "rationale": "Contains factual misattributions and includes an atypical interdisciplinary field (physics). Cut off mid-sentence ('founded as an academic discipline in 195'), limiting instruction following."
+      }
+    },
+    "3": {
+      "baseline": {
+        "factual_errors": [
+          "육식 밥과 함께 먹는다는 잘못된 설명 (김치는 다양한 밥과 함께 먹으며 '육식 밥'은 부적절함)",
+          "잡음료로 만든다는 잘못된 주장 (김치는 발효 채소 요리로 음료가 아님)"
+        ],
+        "factuality": 0,
+        "instruction_following": 0,
+        "language_fluency": 2,
+        "rationale": "김치를 잡음료로 설명하는 등 사실과 전혀 다른 내용이며 문장이 반복되고 불완전하다."
+      },
+      "finetuned": {
+        "factual_errors": [
+          "채소를 삶는다는 잘못된 설명 (전통 김치는 채소를 소금에 절이는 과정을 거침)",
+          "간장 소스로 설명하는 잘못 (김치 양념은 간장이 아닌 고추가루, 젓갈 등으로 만듦)"
+        ],
+        "factuality": 3,
+        "instruction_following": 3,
+        "language_fluency": 3,
+        "rationale": "양배추 등 채소를 언급했으나 삶는 과정과 간장 소스 설명이 부정확하며 문장이 불완전하다."
+      },
+      "pretrained": {
+        "factual_errors": [
+          "칠면조를 재료로 사용한다는 잘못된 주장 (전통 김치에는 칠면조가 들어가지 않음)",
+          "삶아 만든다는 잘못된 설명 (김치는 발효과정을 거치며 삶는 것이 일반적이지 않음)"
+        ],
+        "factuality": 1,
+        "instruction_following": 2,
+        "language_fluency": 3,
+        "rationale": "한국 전통 음식으로 설명하려 했으나 칠면조 재료와 삶는 과정 등 사실 오류가 있으며 설명이 불완전하다."
+      }
+    },
+    "4": {
+      "baseline": {
+        "factual_errors": [
+          "Incorrect population claim ('1/3 이상 of Koreans live in Seoul'; actual ~19%), typo '대기 과' (correct: '대기업과' meaning 'large companies')"
+        ],
+        "factuality": 2,
+        "instruction_following": 3,
+        "language_fluency": 3,
+        "rationale": "Attempts to cover key aspects (capital, economic center) but contains significant factual errors (population proportion, typo leading to incorrect term). "
+      },
+      "finetuned": {
+        "factual_errors": [],
+        "factuality": 4,
+        "instruction_following": 4,
+        "language_fluency": 3,
+        "rationale": "Accurately states Seoul as capital, most populous city, and center of history/culture/economy; minor fluency issue with incomplete final sentence."
+      },
+      "pretrained": {
+        "factual_errors": [
+          "Redundant repetition of '서울역은 서울특별시 중심부에 위치한 서울역이 있습니다', typo '대한민 의' (correct: '대한민국의'), incomplete final sentence"
+        ],
+        "factuality": 1,
+        "instruction_following": 0,
+        "language_fluency": 1,
+        "rationale": "Dominated by repetition, typos, and incomplete sentences; fails to provide a meaningful introduction to Seoul."
+      }
+    },
+    "5": {
+      "baseline": {
+        "factual_errors": [],
+        "factuality": 5,
+        "instruction_following": 5,
+        "language_fluency": 5,
+        "rationale": "Comprehensive explanation including tradition origin (Pilgrims/Native Americans), preparation, sides, and symbolism; highly fluent and fully addresses the task."
+      },
+      "finetuned": {
+        "factual_errors": [],
+        "factuality": 5,
+        "instruction_following": 5,
+        "language_fluency": 5,
+        "rationale": "Accurately explains Thanksgiving turkey as a traditional bird, preparation (roasted/baked, seasoned), and sides; fluent and fully follows the task."
+      },
+      "pretrained": {
+        "factual_errors": [],
+        "factuality": 5,
+        "instruction_following": 2,
+        "language_fluency": 2,
+        "rationale": "Contains redundant phrases, an off-topic 'Explanation' section about the response itself, and incomplete sentences; partially starts explaining but veers off task with poor fluency."
+      }
+    }
+  },
+  "schema_version": "exp8-5-summary-v1",
+  "scientific_findings": {
+    "english_drop": 0.8333,
+    "english_retention_tolerance": 1.0,
+    "english_retention_within_tolerance": true,
+    "kimchi_factual_failure_observed": true,
+    "kimchi_finetuned_factual_errors": [
+      "채소를 삶는다는 잘못된 설명 (전통 김치는 채소를 소금에 절이는 과정을 거침)",
+      "간장 소스로 설명하는 잘못 (김치 양념은 간장이 아닌 고추가루, 젓갈 등으로 만듦)"
+    ],
+    "korean_gain": 1.7777,
+    "korean_gain_observed": true
+  },
+  "stage_averages": {
+    "baseline": {
+      "english": {
+        "factuality": 5.0,
+        "instruction_following": 5.0,
+        "language_fluency": 5.0,
+        "overall": 5.0
+      },
+      "korean": {
+        "factuality": 1.0,
+        "instruction_following": 1.6667,
+        "language_fluency": 2.3333,
+        "overall": 1.6667
+      }
+    },
+    "finetuned": {
+      "english": {
+        "factuality": 4.0,
+        "instruction_following": 4.5,
+        "language_fluency": 4.0,
+        "overall": 4.1667
+      },
+      "korean": {
+        "factuality": 3.6667,
+        "instruction_following": 3.6667,
+        "language_fluency": 3.0,
+        "overall": 3.4444
+      }
+    },
+    "pretrained": {
+      "english": {
+        "factuality": 4.0,
+        "instruction_following": 2.5,
+        "language_fluency": 3.0,
+        "overall": 3.1667
+      },
+      "korean": {
+        "factuality": 1.3333,
+        "instruction_following": 1.0,
+        "language_fluency": 1.6667,
+        "overall": 1.3333
+      }
+    }
+  },
+  "status": "passed",
+  "test_names": {
+    "1": "Korean Wikipedia - Artificial Intelligence (인공지능)",
+    "2": "English Wikipedia - Artificial Intelligence (Preservation Check)",
+    "3": "Korean Instruction - Explain Kimchi",
+    "4": "Korean Instruction - Introduce Seoul",
+    "5": "English Instruction - Thanksgiving Turkey (Preservation Check)"
+  }
+}