liqiang/ai-agent-book
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
Details
dependency resolution / resolve (3.11) (push) Canceled after 0s
Details
dependency resolution / resolve (3.13) (push) Canceled after 0s
Details
deploy-pages / build (push) Canceled after 0s
Details
deploy-pages / deploy (push) Canceled after 0s
Details
i18n consistency check / check (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
Details
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
Details
web-search-agent tests / test (push) Canceled after 0s
Details
web-search-agent tests / agentbook (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
liqiang
2026-08-20 13:12:50 +00:00
commit b119135836
10275 changed files with 3284984 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chapter8/MiniMind-pretrain/validation/runs/exp8-4-training-report-20260731-v1/manifest.json
+83
View File
@@ -0,0 +1,83 @@
{
"acceptance": {
"all_64_historical_outputs_retained": true,
"all_eight_configuration_cells_retained": true,
"checkpoints_not_an_acceptance_artifact": true,
"eight_image_aware_arm_blind_judgments": true,
"future_reproduction_commands_declared": true,
"historical_provenance_limitations_explicit": true,
"historical_report_content_hashed": true,
"immutable_dataset_clip_and_eval_image_inputs_frozen": true,
"immutable_original_and_improved_source_revisions_frozen": true,
"passed": true,
"raw_judge_requests_responses_ids_usage_latency_retained": true,
"request_images_match_pinned_sha256": true,
"same_eight_images_present_in_every_cell": true
},
"artifacts": [
{
"bytes": 22282,
"path": "retained_outputs.json",
"sha256": "6333571fcdb95a84bece9fdf65456f328e7e048342ede37f71e3e03c68c38df6"
},
{
"bytes": 10758,
"path": "reproduction_contract.json",
"sha256": "e530aa559f940a2034e79315432789bf017abf0fc247adba3fb5e802893ea1d7"
},
{
"bytes": 2596470,
"path": "judge_receipts.json",
"sha256": "c777a01a7a484d7a6184575be101c167c0007442144e2bcb1f7047cee0dbc5f0"
},
{
"bytes": 33310,
"path": "summary.json",
"sha256": "fa904dfee4e8dbdf0c9986db79b3eba18adc00b3bfd823e9a99fca10e69a3f6d"
},
{
"bytes": 2116,
"path": "report.md",
"sha256": "0906af01fcf1f6c11528215fe8767c6fd5cad0743286cb0cf94af0df300f3df6"
}
],
"checkpoint_policy": "not distributed; not an acceptance artifact",
"created_at": "2026-08-17T05:34:53.397311+00:00",
"experiment": "8-4",
"inputs": [
{
"bytes": 155255,
"path": "chapter8/MiniMind-pretrain/README.md",
"sha256": "a8e1df1a9ee5cf013995e9ff3b963621485a838c97456b9f30e65ea9fdf55d50"
},
{
"bytes": 14150,
"path": "pyproject.toml",
"sha256": "33cc27c2759f353663d6907f1a918a41a49fb8d3e4e28370d508136449c3b156"
},
{
"bytes": 1311181,
"path": "uv.lock",
"sha256": "347fc87f40526372c284c61e5374536ce8d2071936cef8844d350470ecdf0d0b"
},
{
"bytes": 42148,
"path": "chapter8/MiniMind-pretrain/validation/run_vlm_training_report_audit.py",
"sha256": "d59e8175ce19a1973005263f6f7e4642e1ee11402860b4fadf6a1fd192d4cc7d"
},
{
"bytes": 14797,
"path": "chapter8/MiniMind-pretrain/validation/validate_vlm_evidence.py",
"sha256": "4ce53d7000d7f45a8e8012db1989374fcf42f5f832fbe134aca098e826b95db5"
},
{
"bytes": 6800,
"path": "chapter8/MiniMind-pretrain/validation/test_vlm_training_report_audit.py",
"sha256": "6259638aae28d1035f9d597f8f5ea17c7a66bd6ba8e3759ed1c842c94c19d1e1"
}
],
"run_dir": "validation/runs/exp8-4-training-report-20260731-v1",
"run_id": "exp8-4-training-report-20260731-v1",
"schema_version": "exp8-4-manifest-v1",
"status": "passed"
}
chapter8/MiniMind-pretrain/validation/runs/exp8-4-training-report-20260731-v1/report.md
+26
View File
@@ -0,0 +1,26 @@
# Experiment 8-4 retained-training-report audit
## Result
Status: **passed**. The historical report retains 64 image descriptions across 8 configurations and the same 8 images. Each image was inspected by a real image-capable ARK judge together with all eight arm-blind captions.
| Configuration | Grounding | Hallucination control | Coverage | Specificity | Overall | Best count |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| without_muon_sft | 2.1250 | 1.3750 | 2.3750 | 1.7500 | 1.9062 | 1 |
| without_muon_pretrained | 1.8750 | 2.6250 | 1.5000 | 0.8750 | 1.7188 | 1 |
| muon_from_dpo_sft | 1.3750 | 1.1250 | 1.8750 | 1.7500 | 1.5312 | 2 |
| muon_from_sft_pretrained | 1.8750 | 2.0000 | 1.3750 | 0.8750 | 1.5312 | 2 |
| muon_from_sft_sft | 1.2500 | 1.1250 | 1.5000 | 1.2500 | 1.2812 | 1 |
| muon_from_pretrain_sft | 1.1250 | 0.5000 | 1.3750 | 1.1250 | 1.0312 | 0 |
| muon_from_dpo_pretrained | 1.0000 | 1.5000 | 0.8750 | 0.3750 | 0.9375 | 0 |
| muon_from_pretrain_pretrained | 0.7500 | 1.3750 | 0.7500 | 0.6250 | 0.8750 | 1 |
The highest descriptive judge mean was **without_muon_sft** at **1.9062**. Averaged across all four base configurations, full VLM SFT changed the score by **+0.1718** versus projection-only pretraining.
The isolated report comparison pairs original/SFT-base against QK-Norm+Muon/SFT-base at each VLM stage. QK-Norm and Muon still change together, so no Muon-only causal claim is made. All author-written qualitative claims remain historical observations rather than pass/fail gates.
## Provenance and reproduction boundary
`reproduction_contract.json` freezes separate pre-QK-Norm and QK-Norm+Muon MiniMind-V revisions, the corresponding base-LLM revisions, script-compatible VLM dataset Git-LFS objects, the CLIP weight object, all eight evaluation-image hashes, and future commands. These pins are not misrepresented as the historical checkout.
Training checkpoints remain local by book policy and are not acceptance artifacts. The accepted artifact is this content-hashed report, all 64 retained outputs, eight raw image-aware judge receipts, and explicit limitations.
chapter8/MiniMind-pretrain/validation/runs/exp8-4-training-report-20260731-v1/reproduction_contract.json
+178
View File
@@ -0,0 +1,178 @@
{
"checkpoint_policy": {
"acceptance_artifact": false,
"distributed_with_book": false,
"reason": "Training checkpoints are intentionally not distributed to readers.",
"required_artifact": "reproducible evidence-backed training report"
},
"experiment": "8-4",
"future_reproduction": {
"base_llm_source": {
"dependency": "Use the Experiment 8-3 data/commands to produce original-SFT and improved pretrain/SFT/DPO 768-dimension base checkpoints.",
"original_files_sha256": {
"eval_model.py": "b9f7ea9d7f517551362bbf2da8f1de006b8c734bcba774b2be752bc63cc4349d",
"model/model_minimind.py": "7cb069cb0cb0dfa123cf11ea394d0001270bc683c0a2dfe4120fc3b861ffc0a4",
"trainer/train_dpo.py": "5e556a3089e43681638cdbf5adafb9d085bb1de5e4ea8da3ee522dfae02e3599",
"trainer/train_full_sft.py": "a57422f1df80bf2867f31f3b4a646a92ac7f66729e98a3b32cd1ec4d6780cb8b",
"trainer/train_pretrain.py": "ddd122645a9f1043bc8dac69a81ac51d2df95df8745d25faed7963d38fedc328"
},
"original_revision": "6d160ea20b98324632c4447ee63ec7cfa9becd20",
"qk_norm_muon_files_sha256": {
"eval_model.py": "43930a4b55048a4a3ffa17eb78ae67d59582d639aa9365f0bbf41ba149128af8",
"model/model_minimind.py": "2d33988711c704be6a22c4c61489b23106a2340a7cb8b97ebe3e40f30819cbb0",
"trainer/train_dpo.py": "97f2c31cc8bc21a777e2efcb5e2fa35a49e4e9e3698db120148f8a0b2f678449",
"trainer/train_full_sft_muon.py": "acd0b7db5b1d8b25d3c3103f92d68a7d381f9322a1b33bbec34be3d005930bad",
"trainer/train_pretrain_muon.py": "fc83d07754ec3a8c156b6b8bfc0fd4326edecb72efabc5e08ae4ff5e3a7029bc"
},
"qk_norm_muon_revision": "8bdc5d97d5845a8c1ac2ed56a5b8b4c0d0fb0795",
"repository": "bojieli/minimind"
},
"commands": {
"dataset": "git clone https://huggingface.co/datasets/jingyaogong/minimind-v_dataset dataset-source && git -C dataset-source checkout --detach ac9d03a3fd26a2d8e74bda374d9a2ddba49e4c1b && cp dataset-source/{pretrain_data.jsonl,sft_data.jsonl} dataset/ && unzip dataset-source/pretrain_images.zip -d dataset && unzip dataset-source/sft_images.zip -d dataset",
"evaluation": "For every isolated out_dir, preserve both checkpoints, copy the selected *_muon_768.pth name to eval_vlm.py's pretrain_vlm_768.pth or sft_vlm_768.pth compatibility name when needed, then run python eval_vlm.py --load 0 --model_mode 0 and --model_mode 1 on the eight hash-pinned images with seed 1337.",
"improved_matrix": "For each BASE in pretrain,sft,dpo, install the corresponding Experiment-8-3 QK-Norm+Muon 768-dimension checkpoint as runs/muon-from-$BASE/out/llm_768.pth, then run train_pretrain_vlm_muon.py and train_sft_vlm_muon.py with the same four-epoch data arguments in that isolated out_dir.",
"improved_source": "git clone https://github.com/bojieli/minimind-v.git sources/qk-norm-muon-minimind-v && git -C sources/qk-norm-muon-minimind-v checkout --detach ead791c530fa5f9a3549dbfe9e11ec732d18d2e5",
"original_pretrain_vlm": "install -m 0644 <exp8-3-original-sft-768.pth> runs/original/out/llm_768.pth && cd trainer && torchrun --nproc_per_node=8 train_pretrain_vlm.py --out_dir ../runs/original/out --epochs 4 --hidden_size 768 --num_hidden_layers 16 --data_path ../dataset/pretrain_data.jsonl --images_path ../dataset/pretrain_images --use_wandb",
"original_sft_vlm": "cd trainer && torchrun --nproc_per_node=8 train_sft_vlm.py --out_dir ../runs/original/out --epochs 4 --hidden_size 768 --num_hidden_layers 16 --data_path ../dataset/sft_data.jsonl --images_path ../dataset/sft_images --use_wandb",
"original_source": "git clone https://github.com/bojieli/minimind-v.git sources/original-minimind-v && git -C sources/original-minimind-v checkout --detach 765908051d0837d60cecfb93f8390334e2e55f1e",
"vision_encoder": "git clone https://huggingface.co/openai/clip-vit-base-patch16 model/vision_model/clip-vit-base-patch16 && git -C model/vision_model/clip-vit-base-patch16 checkout --detach 57c216476eefef5ab752ec549e440a49ae4ae5f3"
},
"environment": {
"book_lock": "uv.lock",
"book_pyproject": "pyproject.toml",
"boundary": "The book lock freezes a future Python environment; CUDA, drivers, and the historical GPU image were not retained.",
"install": "uv sync --locked --python 3.12 --extra ch7 --extra dev"
},
"evaluation_images": {
"Astronaut-Space.jpg": {
"sha256": "f466cdafecbdb85d2bad586896db5db3313afe18f9b3505667756cd25b747747",
"source_filename": "太空宇航员-Astronaut-Space.jpg"
},
"Bicycle-Flowers.jpg": {
"sha256": "44fae0fafcd52c20b9bcaded897facbff00f61019cdd0aea543addf8499ad899",
"source_filename": "自行车鲜花-Bicycle-Flowers.jpg"
},
"Chair-Elderly-Reading.jpg": {
"sha256": "8fe91a90e837c33230d21cfe7ba5020e71b3ae99ac4c3fbd6d32cb54f51def53",
"source_filename": "椅子老人看书-Chair-Elderly-Reading.jpg"
},
"Dog-Woman-Sea.jpg": {
"sha256": "ba90d8b8738a44eac70811be5c89f767492b167ad4f6f6c31aa4591837d7e3dc",
"source_filename": "小狗美女海边-Dog-Woman-Sea.jpg"
},
"Panda-Grassland.jpg": {
"sha256": "0b7610a881039f0effdbfa46e9bb189132443d3ce2956856e8adf66d1ca22f8c",
"source_filename": "熊猫草地-Panda-Grassland.jpg"
},
"Rainbow-Falls.jpg": {
"sha256": "1c8b74debaceb2e0bb6171b182084afe49288a0cc8089eb91eac69d067c27b10",
"source_filename": "彩虹瀑布-Rainbow-Falls.jpg"
},
"city-traffic.jpg": {
"sha256": "73e90d82fbc5b1cf43b40de782b443f93f43a34e66b8ddebf3146d5dc1f83e00",
"source_filename": "城市车水马龙-city-traffic.jpg"
},
"dance.jpg": {
"sha256": "939e3132c8d3aec81f66f8aa928b476aaa25e00d94f1097f4974e73c913d5d8c",
"source_filename": "舞蹈-dance.jpg"
}
},
"vision_encoder": {
"file": {
"bytes": 598641023,
"lfs_sha256": "ec89c7b09c749a60aae3c9cd910516f24b58214a7df060b48962d14c469cfbf0",
"path": "pytorch_model.bin"
},
"repository": "openai/clip-vit-base-patch16",
"revision": "57c216476eefef5ab752ec549e440a49ae4ae5f3"
},
"vlm_dataset": {
"files": {
"pretrain_data.jsonl": {
"bytes": 134315765,
"lfs_sha256": "abc9f2ba44190646692fbe7e2b49c366c5045490989fb32d2c5e960dd0ee10e4"
},
"pretrain_images.zip": {
"bytes": 2614907051,
"lfs_sha256": "64d56cee145bed75bc7f94c9cbf58882c41c4a0fea993014e27de7490b49e8b7"
},
"sft_data.jsonl": {
"bytes": 173137988,
"lfs_sha256": "c1993d38c3a22a8bdfee65affc82d6559e5bb62e785b0f21c9151c75116151fc"
},
"sft_images.zip": {
"bytes": 1026332147,
"lfs_sha256": "89ee34facc6793c51613613e0b10cac078942282f5fdec48d85751c6224bc3c2"
}
},
"repository": "jingyaogong/minimind-v_dataset",
"revision": "ac9d03a3fd26a2d8e74bda374d9a2ddba49e4c1b",
"selected_for_jsonl_script_compatibility": true
},
"vlm_source": {
"not_claimed_as_historical_revisions": true,
"original_files_sha256": {
"dataset/lm_dataset.py": "df20d57460d2845841ddf2e0faced1af1f7ec169e7fda3cd50fe3b2854288a92",
"eval_vlm.py": "9d883e4adbab0a7b88fd0cb9034132559a365387ec273ac4811cdd5ad28d5cda",
"model/model_minimind.py": "105429e93dcbe87145264d72d46a6add7639666036e999628c76ae50582507dc",
"model/model_vlm.py": "4ee42b298db68f30fbfa06d7686aa375d41a697628c770d0a134bca40ca9ea80",
"model/tokenizer.json": "d98595c6aef70d95f72748582fb9b4f53d76dd58c1ae1dd702ad7c84e1caf5e4",
"model/tokenizer_config.json": "dbbdb7eea33aba5c2608471494c93f650a2cf46fbe4a7489e531537ddadee746",
"requirements.txt": "a9bddf49d3ccbc9f8a2508ea039aebc0b996dccb0d3618d1b119af53a5d49869",
"trainer/train_pretrain_vlm.py": "4d30d54a940ae2eced204971cc03aafb3eb41a5c84c9f033cfdf162e63924a4d",
"trainer/train_sft_vlm.py": "8e3b920a6a135eb126bbeea07e2db748729cdd86050925b282a80537bc324e5f"
},
"original_revision": "765908051d0837d60cecfb93f8390334e2e55f1e",
"qk_norm_muon_files_sha256": {
"dataset/lm_dataset.py": "df20d57460d2845841ddf2e0faced1af1f7ec169e7fda3cd50fe3b2854288a92",
"eval_vlm.py": "9d883e4adbab0a7b88fd0cb9034132559a365387ec273ac4811cdd5ad28d5cda",
"model/model_minimind.py": "4771bc4b2ac367a6e6415c42c30bcdb54bec0397708f87de3c390042680b1e9e",
"model/model_vlm.py": "4ee42b298db68f30fbfa06d7686aa375d41a697628c770d0a134bca40ca9ea80",
"model/tokenizer.json": "e489029175fb3f94b8211a120a72a2ee41a664db65b828d077c7bde989c845a9",
"model/tokenizer_config.json": "190cc4738bac3b6f6b563376019c581b320fdb0260a03b9d5ab806296c8c6bb8",
"requirements.txt": "a9bddf49d3ccbc9f8a2508ea039aebc0b996dccb0d3618d1b119af53a5d49869",
"trainer/muon.py": "00c2c6a225edeb55433df0724c3c74f6ff98ac4b2cc73c4aafcff686824f6267",
"trainer/train_pretrain_vlm_muon.py": "f39af354c588747d9d5e522c9374a7f59a35d57aa649da74957da67d95d25bc6",
"trainer/train_sft_vlm_muon.py": "1fd56b3e8bed2714b4d10ceba5d57ada0b95d00dfbfb514481498fef0c0dd03d"
},
"qk_norm_muon_revision": "ead791c530fa5f9a3549dbfe9e11ec732d18d2e5",
"repository": "bojieli/minimind-v"
}
},
"historical_evidence_boundary": {
"claim": "The author's report establishes that eight VLM configurations were evaluated on eight images. It does not establish byte identity of the historical code, datasets, base/VLM checkpoints, or RNG state.",
"eight_historical_cells_and_64_outputs_retained": true,
"historical_base_checkpoint_hashes_retained": false,
"historical_dataset_hashes_retained": false,
"historical_rng_and_stepwise_logs_retained": false,
"historical_source_revisions_retained": false,
"historical_training_executed": true,
"historical_vlm_checkpoint_hashes_retained": false
},
"reported_training_design": {
"base_llm_stages": [
"pretrain",
"sft",
"dpo"
],
"parameter_count_millions": {
"original": 104.622,
"qk_norm_muon": 104.625
},
"projection_pretraining_freezes_llm": true,
"reported_epochs": 4,
"seed_in_current_source": 1337,
"sft_unfreezes_full_model": true,
"source_verified_mechanisms": {
"improved_revision_has_qk_norm_before_rope": true,
"improved_revision_uses_muon_for_selected_2d_weights": true,
"original_revision_precedes_qk_norm_commit": true,
"vision_encoder_is_frozen_clip": true
},
"vlm_stages": [
"pretrained",
"sft"
]
},
"schema_version": "exp8-4-reproduction-contract-v1"
}
chapter8/MiniMind-pretrain/validation/runs/exp8-4-training-report-20260731-v1/retained_outputs.json
+366
View File
@@ -0,0 +1,366 @@
{
"cell_count": 8,
"cells": [
{
"architecture": "original",
"base_llm_stage": "sft",
"config": "without_muon_pretrained",
"historical_command": "$ python eval_vlm.py --load 0 --model 0",
"output_count": 8,
"outputs": [
{
"image": "Rainbow-Falls.jpg",
"output": "Water droplets on the rainbow bridge"
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "A young person decorating the beach with a dog's head."
},
{
"image": "dance.jpg",
"output": "This dancer is performing on stage."
},
{
"image": "Astronaut-Space.jpg",
"output": "An astronaut is performing a spacewalk in space."
},
{
"image": "city-traffic.jpg",
"output": "On the city streets at nightfall."
},
{
"image": "Panda-Grassland.jpg",
"output": "In the zoo, a panda is eating bamboo."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "Using old bicycle tires and wheels on an old bicycle, adding flowers and decorations to the bicycle."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "An elderly woman reading a book in the park."
}
],
"vlm_stage": "pretrained"
},
{
"architecture": "original",
"base_llm_stage": "sft",
"config": "without_muon_sft",
"historical_command": "$ python eval_vlm.py --load 0 --model 1",
"output_count": 8,
"outputs": [
{
"image": "Rainbow-Falls.jpg",
"output": "This picture depicts a huge, colorful giant waterfall located next to a highway. From the image, you can see a massive and majestic giant waterfall. A giant rainbow flag hangs above the waterfall, adding visual appeal. The waterfall is surrounded by beautiful scenery, adding beauty to this picturesque landscape."
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "This image depicts a woman sitting on a beach with her dog beside her, while her dog sits on a blue and white checkered blanket."
},
{
"image": "dance.jpg",
"output": "The picture depicts a woman performing a dance on stage, her dance steps soaring high above the stage. Several chairs are placed on the stage, one on the left side of the frame, another on the right, and one more on the right side.\nMultiple chairs are placed on the stage, one near the left, another on the right. There is also a clock on the stage, possibly to help the performer keep time."
},
{
"image": "Astronaut-Space.jpg",
"output": "The image shows a huge spaceship that looks like a giant commercial airplane. This airplane is in the center of the frame, flying over a small hill and a barn. The airplane looks like a hangar, filled with all sorts of things.\nAround the airplane, some people might be observing this large commercial aircraft. Some of them seem to be watching the pilot's actions, while others might be admiring the details of the aircraft or enjoying the scene."
},
{
"image": "city-traffic.jpg",
"output": "This picture depicts a busy city street with heavy traffic, several cars and a truck moving. Tall buildings and streetlights illuminate the area, creating a vibrant and lively atmosphere.\nOn the street, various vehicles can be seen, including several cars and a truck. Some pedestrians also appear, possibly going about their daily activities or navigating the urban environment.\nOverall, this image captures a vibrant urban environment where vehicles, pedestrians, and traffic lights together create a lively atmosphere."
},
{
"image": "Panda-Grassland.jpg",
"output": "The image depicts a black and white panda wearing glasses, a long-haired black and white panda sitting on the grass. This panda seems to be staring at the camera, it looks like it is looking at the camera's image. This panda seems to be lying on the grass resting, possibly to avoid the sun or rain."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "The image shows a bicycle parked outside a building, with a row of vases and bouquets visible in the background. The bicycle occupies a considerable portion of the frame, extending from left to right. Throughout the scene, there are three potted plants, one near the left, another on the right, and a third on the left.\nBesides the bicycle, there are two people in the scene, one standing on the right and the other near the left side of the image."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "The image depicts an elderly man wearing glasses sitting on a park bench. He seems to be reading a book, possibly enjoying the outdoor environment. Several cars are parked nearby, one in the center of the image and another on the right. There is also a bench in the background, right next to the person sitting on the bench."
}
],
"vlm_stage": "sft"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "dpo",
"config": "muon_from_dpo_pretrained",
"historical_command": "$ python eval_vlm.py --load 0 --model 0",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "Night streets with elevators and sidewalks."
},
{
"image": "Astronaut-Space.jpg",
"output": "Spaceman sent to a new Earth"
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "You have a small ball in your palm, you throw it into the water."
},
{
"image": "Rainbow-Falls.jpg",
"output": "Viewed on the water, water landscape"
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "An elderly person sitting on a park bench."
},
{
"image": "Panda-Grassland.jpg",
"output": "This photo captures the cuteness and loveliness of this little white bear."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "Riding a bicycle on the street, bicycles everywhere, I saw many cars, this is my bicycle"
},
{
"image": "dance.jpg",
"output": "Performed by actors performing on stage"
}
],
"vlm_stage": "pretrained"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "dpo",
"config": "muon_from_dpo_sft",
"historical_command": "$ python eval_vlm.py --load 0 --model 1",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "This picture depicts a busy city street filled with skyscrapers and vehicles. At least twelve cars are driving on the street, some parked on the side, while others are moving.\nSeveral traffic lights can be seen, one on the left side of the street, another in the middle, and one on the right. Additionally, a bus is heading towards the center of the frame, adding to the lively atmosphere of this bustling city."
},
{
"image": "Astronaut-Space.jpg",
"output": "The person in the picture is wearing a black spacesuit and glasses, sitting on what looks like a space shuttle. They seem to be looking at a space shuttle flying in front of them, which is located on the right side of the scene.\nBesides the space shuttle, there are two other small boats in the picture, one on the left and another on the right. A smaller plane can also be seen in the background, and the person is located on the far right."
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "The image depicts a woman sitting on a bench by the beach, holding a small dog outside to sunbathe. She is wearing a dress and seems to be enjoying the outdoors.\nIn the background, several other people can be seen scattered on the beach, some closer to the water and others farther away. They all look happy, possibly spending a good time with this woman."
},
{
"image": "Rainbow-Falls.jpg",
"output": "The image shows a beautiful fountain surrounded by mist above it. The fountain is located above a high mountain, adding a magnificent background to the scene. An umbrella can be seen on the water, providing shade for visitors.\nThe terrain around the fountain consists of hills and mountains, offering a picturesque landscape for visitors. The presence of the fountain, water features, and surrounding mountains creates a serene and awe-inspiring atmosphere, making the area an ideal resort destination for photography enthusiasts and outdoor lovers."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "The image shows an elderly man sitting on a park bench, sitting in front of a book, holding it in his hands. The person looks relaxed and casual, possibly enjoying the outdoors.\nThere are several trees in the park, adding a natural atmosphere to the scene. Multiple benches are placed in different locations in the park for people to sit and enjoy the surroundings."
},
{
"image": "Panda-Grassland.jpg",
"output": "The protagonist of the picture is a giant panda with black and white markings, standing on a lush, green grassland. This giant panda seems to be enjoying the warmth of the sun and the grass."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "The image shows a green bicycle parked on a lane with yellow flowers, loaded with colorful blossoms. The bicycle is parked in front of a building, with two adjacent buildings forming a stark contrast.\nBesides the bicycle and flowers, there is a cup placed on the ground, possibly on or near the sidewalk."
},
{
"image": "dance.jpg",
"output": "In the image, a woman in a black dress is standing on a stage. She is performing on stage, surrounded by several people watching her, some of whom are holding mobile phones. These phones are scattered throughout the scene, possibly taken to capture the performance or communicate with the audience."
}
],
"vlm_stage": "sft"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "pretrain",
"config": "muon_from_pretrain_pretrained",
"historical_command": "$ python eval_vlm.py --load 0 --model 0",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "A bus traveling from the city center to a tall building"
},
{
"image": "Astronaut-Space.jpg",
"output": "🤖️: A little boy on a spaceship"
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "A dog walking by the seaside"
},
{
"image": "Rainbow-Falls.jpg",
"output": "Water droplets falling from a cave"
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "A young man reading a book on an elderly person's bed."
},
{
"image": "Panda-Grassland.jpg",
"output": "A bear with a layer of cute black-and-white fur"
},
{
"image": "Bicycle-Flowers.jpg",
"output": "In a park in Paris, a little boy rides a bicycle along the riverbank."
},
{
"image": "dance.jpg",
"output": "Performers wearing colorful costumes and fashionable outfits during a performance."
}
],
"vlm_stage": "pretrained"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "pretrain",
"config": "muon_from_pretrain_sft",
"historical_command": "$ python eval_vlm.py --load 0 --model 1",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "This image depicts a bustling city street at night, with high-rise buildings and heavy traffic. Cars are driving in different directions, creating a vibrant urban scene. Cars, trucks, and motorcycles flow continuously, showing the bustling activity on this street.\nA tall building stands prominently in the frame, forming part of the city skyline. The unique architectural style and scale of this building catch the attention of passersby, who might be drawn to appreciate its aesthetic value."
},
{
"image": "Astronaut-Space.jpg",
"output": "The image features a large spaceship or spacecraft, equipped with electronic devices and flight instruments. It appears to be from a museum or exhibition space, as various objects are displayed inside. These objects vary in size and seem to have been created by different professionals or scientists for visitors to explore."
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "In the picture, a woman sits on a beach bench holding a brown-and-white puppy. The puppy looks very happy, enjoying its time on the sand. In the background, two other people can be seen, also sitting on the same bench.\nBesides the woman and the puppy, various items appear in the scene, such as two handbags, a bench near the dog, a chair on the left, and a chair near the center of the scene."
},
{
"image": "Rainbow-Falls.jpg",
"output": "This image shows a stunning scene of rough seas extending to the horizon under a cloudy sky. A large mountain with a white dome stands in the center of the frame, becoming the focal point. The sea seems to be crashing against the horizon, creating an impression of strong waves.\nIn this scene, several people are scattered around, possibly enjoying the sea and the spectacular sky. Some of these people are closer to the coast, while others are farther away, likely admiring the breathtaking view."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "An elderly man wearing glasses sits at a table in a library, surrounded by cluttered bookshelves and various books. He is sitting in a chair, focused on a book, possibly meditating or deep in thought. Behind him, a potted plant can be seen, adding a touch of greenery to the scene.\nThe library environment is very crowded, with bookshelves filled with all kinds of books, covering almost the entire wall. These books vary in size and position, some even overturned, giving the library a busy atmosphere."
},
{
"image": "Panda-Grassland.jpg",
"output": "In the picture, a large black-and-white bear sits on green grass, with a few flowers scattered around. It seems to be enjoying time outdoors, possibly in a grove or on the grassland."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "The picture shows an outdoor garden with a bicycle parked between two other bicycles. Each bicycle has a basket filled with bright flowers. These flowers are arranged in various vases, adding color and vitality to the overall scene.\nIn the background, several people are in the garden, possibly enjoying a nice day. Some of them are near the bicycles, while others are elsewhere in the garden. Besides the bicycles, two cars are parked nearby, one near the foreground and the other farther away."
},
{
"image": "dance.jpg",
"output": "This image depicts a woman in elegant clothing walking down a wide walkway in the dark. She seems to be attending a dance, as she holds an umbrella, possibly to shield from rain or sunlight.\nChairs are placed along the walkway, likely for the audience to sit and enjoy the dance. Several kites can be seen in the picture, adding a joyful atmosphere to the dance."
}
],
"vlm_stage": "sft"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "sft",
"config": "muon_from_sft_pretrained",
"historical_command": "$ python eval_vlm.py --load 0 --model 0",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "Monitoring of nighttime traffic lights"
},
{
"image": "Astronaut-Space.jpg",
"output": "A series of conversations between \"our ship\" and \"our ship.\""
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "A cute woman in a white long dress and a cute pet dog"
},
{
"image": "Rainbow-Falls.jpg",
"output": "From the mountaintop, water glistens in the sunlight."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "An elderly person sitting on a bench in the park"
},
{
"image": "Panda-Grassland.jpg",
"output": "A cute zebra in the woods"
},
{
"image": "Bicycle-Flowers.jpg",
"output": "This video uses a horse working in a car."
},
{
"image": "dance.jpg",
"output": "A dancer performing on stage, wearing a stylish outfit."
}
],
"vlm_stage": "pretrained"
},
{
"architecture": "qk_norm_muon",
"base_llm_stage": "sft",
"config": "muon_from_sft_sft",
"historical_command": "$ python eval_vlm.py --load 0 --model 1",
"output_count": 8,
"outputs": [
{
"image": "city-traffic.jpg",
"output": "This photo captures a busy city street at night, with cars and trucks on the road. The street is lined with high-rise buildings, creating a bustling atmosphere. The street is crowded with traffic, including multiple cars and a truck. Some of these vehicles are stationary, while others are moving.\nA large number of high-rise buildings appear in the frame, highlighting the urban environment. These buildings vary in size and shape, forming the architectural landscape of the area. This nighttime photo emphasizes the dynamic nature of city life, with various elements combining to create the character of a night-time metropolis."
},
{
"image": "Astronaut-Space.jpg",
"output": "The scene in the image shows a soldier sitting on a blue ship, which contains a large TV screen. The soldier appears to be looking at a smartphone screen, possibly for monitoring or communicating with the outside world.\nThere are several other people on the ship, possibly crew members or scientists. Some of them are scattered around the ship, while others are near the TV screen. Additionally, two TV remote controls are placed on the ship, indicating they might be monitoring or interacting with the equipment."
},
{
"image": "Dog-Woman-Sea.jpg",
"output": "In the photo, a woman sits on the beach holding a large brown dog. She seems to be posing with the dog for the picture. The dog is on the left side of the frame.\nIn the background, multiple figures can be seen scattered across the beach. Some of them appear to be enjoying the beach, while others might be sightseeing or socializing."
},
{
"image": "Rainbow-Falls.jpg",
"output": "This image showcases an impressive fountain landscape, towering high on a grassy hill. The fountain occupies a significant portion of the frame, extending from the center to the far right, looking like a massive work of art.\nThe surroundings of the fountain, including the hill below and some trees in the background, provide a picturesque backdrop for this impressive fountain."
},
{
"image": "Chair-Elderly-Reading.jpg",
"output": "In the picture, an elderly man sits on a park bench reading a newspaper. He seems to be enjoying a peaceful time in the park, perhaps resting in the afternoon or pondering something.\nIn the background, multiple cars can be seen parked near the park, creating a tranquil atmosphere. Additionally, a person appears at the top of the frame, possibly watching the man read the newspaper, perhaps enjoying nature or waiting for someone."
},
{
"image": "Panda-Grassland.jpg",
"output": "The picture shows a small black-and-white panda standing on a patch of grass. This cute panda looks very relaxed and comfortable."
},
{
"image": "Bicycle-Flowers.jpg",
"output": "The picture shows a green bicycle parked next to a bench covered with a green tablecloth. This bench seems designed for gatherings, as it is near a potted plant.\nBesides the bicycle and the potted plant, a car is parked on the right side of the scene. A stop sign can be seen in the background, adding an urban touch to the image."
},
{
"image": "dance.jpg",
"output": "The image shows a woman in a tuxedo standing on a stage, holding a microphone, seemingly delivering an engaging performance. She looks like an actor or performer, with several people around her, possibly the audience or participants.\nVarious items are placed on the stage, including a wine glass and several bottles. Some of these bottles are near the stage, while others are scattered in the background. The scene captures an event on stage, with the actor or performer delivering a memorable show for the audience."
}
],
"vlm_stage": "sft"
}
],
"configs": [
"without_muon_pretrained",
"without_muon_sft",
"muon_from_dpo_pretrained",
"muon_from_dpo_sft",
"muon_from_pretrain_pretrained",
"muon_from_pretrain_sft",
"muon_from_sft_pretrained",
"muon_from_sft_sft"
],
"experiment": "8-4",
"images": [
"Rainbow-Falls.jpg",
"Dog-Woman-Sea.jpg",
"dance.jpg",
"Astronaut-Space.jpg",
"city-traffic.jpg",
"Panda-Grassland.jpg",
"Bicycle-Flowers.jpg",
"Chair-Elderly-Reading.jpg"
],
"output_count": 64,
"schema_version": "exp8-4-retained-outputs-v1",
"source_report": "chapter8/MiniMind-pretrain/README.md",
"source_report_sha256": "a8e1df1a9ee5cf013995e9ff3b963621485a838c97456b9f30e65ea9fdf55d50"
}
chapter8/MiniMind-pretrain/validation/runs/exp8-4-training-report-20260731-v1/summary.json
+858
View File
@@ -0,0 +1,858 @@
{
"acceptance": {
"all_64_historical_outputs_retained": true,
"all_eight_configuration_cells_retained": true,
"checkpoints_not_an_acceptance_artifact": true,
"eight_image_aware_arm_blind_judgments": true,
"future_reproduction_commands_declared": true,
"historical_provenance_limitations_explicit": true,
"historical_report_content_hashed": true,
"immutable_dataset_clip_and_eval_image_inputs_frozen": true,
"immutable_original_and_improved_source_revisions_frozen": true,
"passed": true,
"raw_judge_requests_responses_ids_usage_latency_retained": true,
"request_images_match_pinned_sha256": true,
"same_eight_images_present_in_every_cell": true
},
"best_counts": {
"muon_from_dpo_pretrained": 0,
"muon_from_dpo_sft": 2,
"muon_from_pretrain_pretrained": 1,
"muon_from_pretrain_sft": 0,
"muon_from_sft_pretrained": 2,
"muon_from_sft_sft": 1,
"without_muon_pretrained": 1,
"without_muon_sft": 1
},
"config_averages": {
"muon_from_dpo_pretrained": {
"coverage": 0.875,
"grounding_accuracy": 1.0,
"hallucination_control": 1.5,
"overall": 0.9375,
"visual_specificity": 0.375
},
"muon_from_dpo_sft": {
"coverage": 1.875,
"grounding_accuracy": 1.375,
"hallucination_control": 1.125,
"overall": 1.5312,
"visual_specificity": 1.75
},
"muon_from_pretrain_pretrained": {
"coverage": 0.75,
"grounding_accuracy": 0.75,
"hallucination_control": 1.375,
"overall": 0.875,
"visual_specificity": 0.625
},
"muon_from_pretrain_sft": {
"coverage": 1.375,
"grounding_accuracy": 1.125,
"hallucination_control": 0.5,
"overall": 1.0312,
"visual_specificity": 1.125
},
"muon_from_sft_pretrained": {
"coverage": 1.375,
"grounding_accuracy": 1.875,
"hallucination_control": 2.0,
"overall": 1.5312,
"visual_specificity": 0.875
},
"muon_from_sft_sft": {
"coverage": 1.5,
"grounding_accuracy": 1.25,
"hallucination_control": 1.125,
"overall": 1.2812,
"visual_specificity": 1.25
},
"without_muon_pretrained": {
"coverage": 1.5,
"grounding_accuracy": 1.875,
"hallucination_control": 2.625,
"overall": 1.7188,
"visual_specificity": 0.875
},
"without_muon_sft": {
"coverage": 2.375,
"grounding_accuracy": 2.125,
"hallucination_control": 1.375,
"overall": 1.9062,
"visual_specificity": 1.75
}
},
"experiment": "8-4",
"isolated_original_vs_qk_norm_muon_from_sft": {
"pretrained": {
"delta": -0.1876,
"original": 1.7188,
"qk_norm_muon_from_sft": 1.5312
},
"sft": {
"delta": -0.625,
"original": 1.9062,
"qk_norm_muon_from_sft": 1.2812
}
},
"judge": {
"blind_seed": 740731,
"calls": 8,
"image_aware": true,
"model": "doubao-seed-1-6-250615",
"provider": "ark",
"response_ids": [
"021785497883895355cac89e6d983ae8d30678a5b54f9afc7a30b",
"0217854978839024aa8fafbe9055982037cfb2f5ec29c229036d1",
"021785497883901cbe31712b8a2594f4e458beea699b0faabecc8",
"0217854978838946c88a39b89e3e3930a4a1be3bade8084753bc5",
"0217854979402514aa8fafbe9055982037cfb2f5ec29c22876438",
"021785497944549bdf0d8876ec55e469c432250c926544c9a9a49",
"0217854979562513b2c90331db17702ff1ef9ef62383e60dd8c12",
"02178549797786577f9606cef83fa80b3d313b2e208ca1cdfc167"
],
"total_latency_ms": 557409.335,
"total_tokens": 43094
},
"limitations": [
"Historical base-LLM and VLM checkpoints are intentionally not distributed and were not recreated in this audit.",
"Historical source revisions, dataset identities, RNG state, hardware image, and stepwise logs were not retained.",
"Current immutable pins define a future reproduction contract and are not represented as the exact historical run.",
"The English captions are translations in a bilingual report, so translation can affect judging.",
"One image-aware judge call evaluates all eight anonymous candidates per image; scores are descriptive, not a powered significance test.",
"QK-Norm and Muon change together in the improved arm, so the report does not attribute effects to Muon alone."
],
"per_image_config_scores": {
"Astronaut-Space.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"'sent to a new Earth' is not visible; no indication of a mission to a new Earth"
],
"rationale": "Mentions astronaut ('spaceman') but includes unsupported claim about being 'sent to a new Earth'.",
"visual_specificity": 1
},
"muon_from_dpo_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"spacesuit is white, not black; no glasses; astronaut is standing, not sitting; no boats or plane in image; astronaut is on the left, not far right"
],
"rationale": "Contains multiple incorrect details (color, position, invented objects) and misidentifies actions.",
"visual_specificity": 0
},
"muon_from_pretrain_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"'little boy' is not present; subject is an adult astronaut"
],
"rationale": "Incorrectly identifies subject as a little boy instead of an astronaut.",
"visual_specificity": 0
},
"muon_from_pretrain_sft": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"'museum or exhibition space' is incorrect; scene is inside a functional spacecraft, not a museum; 'objects displayed for visitors' is not visible"
],
"rationale": "Partially mentions spacecraft and instruments but incorrectly claims it's a museum exhibit.",
"visual_specificity": 2
},
"muon_from_sft_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no conversations or dialogue visible; irrelevant to image content"
],
"rationale": "No visual support for conversations; unrelated to the scene.",
"visual_specificity": 0
},
"muon_from_sft_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"subject is an astronaut, not a soldier; no other people present; no smartphone or TV remote controls visible; not a 'blue ship' but a spacecraft interior"
],
"rationale": "Completely misrepresents the scene with invented elements.",
"visual_specificity": 0
},
"without_muon_pretrained": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 3,
"material_errors": [
"astronaut is inside a spacecraft, not performing a spacewalk (no spacewalk visible)"
],
"rationale": "Correctly identifies astronaut but incorrectly claims a spacewalk; astronaut is inside the spacecraft.",
"visual_specificity": 2
},
"without_muon_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no commercial airplane; no small hill or barn; no people observing; central object is a space station, not an airplane"
],
"rationale": "Describes an invented scene with no relation to the actual image.",
"visual_specificity": 0
}
},
"Bicycle-Flowers.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"riding a bicycle (bicycle is parked)",
"bicycles everywhere (only one bicycle)",
"many cars (no cars)",
"this is my bicycle (no ownership indicated)"
],
"rationale": "Contains entirely false claims about riding, multiple bicycles, cars, and personal ownership; no accurate scene elements.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 4,
"grounding_accuracy": 4,
"hallucination_control": 3,
"material_errors": [
"two adjacent buildings (only one building visible)",
"cup placed on the ground (no cup)"
],
"rationale": "Accurately describes green parked bicycle with yellow/colorful flowers in front of a building; minor errors with extra buildings and cup.",
"visual_specificity": 4
},
"muon_from_pretrain_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"park in Paris (no park/Paris context)",
"little boy rides (no boy; bicycle is parked)",
"riverbank (no riverbank)"
],
"rationale": "Completely invented scenario with no relation to the image's bicycle, flowers, or building.",
"visual_specificity": 0
},
"muon_from_pretrain_sft": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"outdoor garden (no garden; sidewalk next to building)",
"two other bicycles (only one bicycle)",
"vases (flowers in baskets, not vases)",
"several people (no people)",
"two cars (no cars)"
],
"rationale": "Incorrectly claims garden setting, multiple bicycles, people, cars, and vases; only bicycle with flower baskets partially accurate.",
"visual_specificity": 1
},
"muon_from_sft_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"video (still image)",
"horse working in a car (no horse or car)"
],
"rationale": "Entirely unrelated to the image; mentions horse and car in a video, neither present.",
"visual_specificity": 0
},
"muon_from_sft_sft": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"bench with green tablecloth (no bench)",
"potted plant (no potted plant)",
"car parked on right (no car)",
"stop sign (no stop sign)"
],
"rationale": "Invents bench, potted plant, car, and stop sign; only green bicycle color is accurate but irrelevant to main scene.",
"visual_specificity": 1
},
"without_muon_pretrained": {
"coverage": 3,
"grounding_accuracy": 3,
"hallucination_control": 4,
"material_errors": [],
"rationale": "Accurately references the bicycle, tires/wheels, and flowers, though lacks details like green color and baskets.",
"visual_specificity": 2
},
"without_muon_sft": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"row of vases (no vases; flowers in baskets)",
"three potted plants (no potted plants)",
"two people (no people)"
],
"rationale": "Mentions bicycle and building but invents vases, potted plants, and people not present in the image.",
"visual_specificity": 1
}
},
"Chair-Elderly-Reading.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 3,
"grounding_accuracy": 5,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Correctly identifies elderly person on park bench with no hallucinations, but lacks reading activity.",
"visual_specificity": 2
},
"muon_from_dpo_sft": {
"coverage": 4,
"grounding_accuracy": 4,
"hallucination_control": 4,
"material_errors": [
"multiple benches not visible"
],
"rationale": "Correctly identifies elderly man on park bench holding a book, trees in background; only error is multiple benches.",
"visual_specificity": 4
},
"muon_from_pretrain_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"young man instead of elderly",
"bed instead of park bench"
],
"rationale": "No young man or bed visible; subject and setting are entirely incorrect.",
"visual_specificity": 0
},
"muon_from_pretrain_sft": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"library instead of park",
"table instead of bench",
"cluttered bookshelves not present",
"potted plant not visible"
],
"rationale": "Incorrect library setting; no table, bookshelves, or potted plant; only elderly man with glasses is correct.",
"visual_specificity": 1
},
"muon_from_sft_pretrained": {
"coverage": 3,
"grounding_accuracy": 5,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Correctly identifies elderly person on park bench with no hallucinations, but lacks reading activity.",
"visual_specificity": 1
},
"muon_from_sft_sft": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"reading newspaper instead of book",
"multiple cars not visible",
"person at top of frame not present"
],
"rationale": "Reads book (not newspaper); no cars or person at top; bench and park are correct but other details invented.",
"visual_specificity": 1
},
"without_muon_pretrained": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 4,
"material_errors": [
"incorrect gender (woman instead of man)"
],
"rationale": "Correct park setting and reading a book, but misidentifies gender as woman.",
"visual_specificity": 2
},
"without_muon_sft": {
"coverage": 3,
"grounding_accuracy": 3,
"hallucination_control": 2,
"material_errors": [
"several cars parked nearby not visible",
"another bench in the background not present"
],
"rationale": "Correct elderly man with glasses reading a book on park bench, but invents cars and another bench.",
"visual_specificity": 3
}
},
"Dog-Woman-Sea.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no ball, palm, or throwing action in image"
],
"rationale": "Completely irrelevant; no ball or throwing present.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 1,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"woman sits on sand, not a bench",
"woman not holding dog; dog sits beside her",
"woman wears sleeveless top/jeans, not a dress",
"no other people in background"
],
"rationale": "Contains multiple hallucinations: bench, held dog, dress, and other people, all absent.",
"visual_specificity": 0
},
"muon_from_pretrain_pretrained": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 4,
"material_errors": [
"dog is sitting, not walking",
"missing woman sitting beside dog"
],
"rationale": "Correct about dog and seaside but misses woman and incorrectly states dog is walking.",
"visual_specificity": 1
},
"muon_from_pretrain_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no beach bench; woman sits on sand",
"woman not holding dog; dog sits beside her",
"dog is adult, not a puppy",
"no other people, handbags, chairs, or bench"
],
"rationale": "Filled with hallucinated objects (bench, handbags, chairs) and incorrect actions (holding puppy).",
"visual_specificity": 0
},
"muon_from_sft_pretrained": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 3,
"material_errors": [
"woman wears light blue sleeveless top and blue jeans, not white long dress"
],
"rationale": "Identifies woman and dog but misrepresents clothing (white long dress vs. blue top/jeans).",
"visual_specificity": 1
},
"muon_from_sft_sft": {
"coverage": 2,
"grounding_accuracy": 1,
"hallucination_control": 1,
"material_errors": [
"woman not holding dog; dog sits beside her",
"no multiple figures in background",
"dog is brown and white, not just brown"
],
"rationale": "Incorrectly claims woman holds dog and background has multiple people; dog color misrepresented.",
"visual_specificity": 1
},
"without_muon_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no young person decorating beach",
"no dog's head; whole dog sits beside woman"
],
"rationale": "Entirely misrepresents scene; no decorating or dog's head present.",
"visual_specificity": 0
},
"without_muon_sft": {
"coverage": 3,
"grounding_accuracy": 4,
"hallucination_control": 4,
"material_errors": [
"dog is not on a blue and white checkered blanket; dog sits on sand"
],
"rationale": "Correctly identifies woman sitting on beach with dog beside her; falsely claims dog sits on a checkered blanket (not present).",
"visual_specificity": 3
}
},
"Panda-Grassland.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"subject is a black-and-white panda, not a white bear"
],
"rationale": "Refers to the panda as a 'white bear'; panda has distinct black-and-white fur.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 4,
"grounding_accuracy": 3,
"hallucination_control": 2,
"material_errors": [
"panda is lying, not standing; no evidence of sun warmth"
],
"rationale": "Accurately identifies giant panda with black and white markings on lush green grassland but incorrectly states it is standing and mentions unevidenced sun warmth.",
"visual_specificity": 4
},
"muon_from_pretrain_pretrained": {
"coverage": 3,
"grounding_accuracy": 4,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Accurately describes the panda as a bear with cute black-and-white fur, with no incorrect details.",
"visual_specificity": 3
},
"muon_from_pretrain_sft": {
"coverage": 3,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"panda is lying, not sitting; no flowers visible; no grove (only grass)"
],
"rationale": "Identifies black-and-white bear on green grass but includes hallucinations (flowers, grove) and incorrect position (sitting).",
"visual_specificity": 2
},
"muon_from_sft_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"subject is panda, not zebra; setting is grass, not woods"
],
"rationale": "Incorrectly identifies subject as zebra and setting as woods; image shows a panda on grass.",
"visual_specificity": 0
},
"muon_from_sft_sft": {
"coverage": 3,
"grounding_accuracy": 2,
"hallucination_control": 3,
"material_errors": [
"panda is lying, not standing"
],
"rationale": "Identifies black-and-white panda and grass but incorrectly states the panda is standing (it is lying).",
"visual_specificity": 2
},
"without_muon_pretrained": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 0,
"material_errors": [
"no evidence of zoo; panda is not eating bamboo (lying on grass)"
],
"rationale": "States panda is in a zoo eating bamboo; image shows panda lying on grass with no bamboo or zoo elements.",
"visual_specificity": 0
},
"without_muon_sft": {
"coverage": 3,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"panda is not wearing glasses; no evidence of sun or rain; conflicting positions (sitting vs lying)"
],
"rationale": "Identifies black-and-white panda on grass and mentions lying/resting but includes hallucinations (glasses, sun/rain) and conflicting position (sitting).",
"visual_specificity": 1
}
},
"Rainbow-Falls.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Vaguely refers to a 'water landscape' but lacks specific details about the waterfall or rainbow.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"fountain is not present (it is a waterfall)",
"umbrella on water is not present",
"visitors are not present"
],
"rationale": "Incorrectly identifies the waterfall as a 'fountain' and adds non-existent elements like umbrella and visitors.",
"visual_specificity": 0
},
"muon_from_pretrain_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"cave is not present",
"water source is a waterfall, not droplets from a cave"
],
"rationale": "Invents a 'cave' and misrepresents the water source; does not depict the waterfall.",
"visual_specificity": 0
},
"muon_from_pretrain_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"rough seas not present",
"cloudy sky not present (sky is clear)",
"mountain with white dome not present",
"people not present"
],
"rationale": "Describes an unrelated seascape with people and mountains, not the waterfall scene.",
"visual_specificity": 0
},
"muon_from_sft_pretrained": {
"coverage": 2,
"grounding_accuracy": 3,
"hallucination_control": 3,
"material_errors": [
"missing mention of waterfall and rainbow",
"perspective as 'from the mountaintop' is unclear"
],
"rationale": "Accurately notes water glistening in sunlight but omits key elements (waterfall, rainbow) and has unclear perspective.",
"visual_specificity": 2
},
"muon_from_sft_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"waterfall is misidentified as a fountain"
],
"rationale": "Falsely labels the waterfall as a 'fountain landscape' with no basis in the image.",
"visual_specificity": 1
},
"without_muon_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"rainbow bridge is not present",
"focus on water droplets is incorrect"
],
"rationale": "Mentions non-existent 'rainbow bridge' and misfocuses on droplets; fails to describe the waterfall.",
"visual_specificity": 0
},
"without_muon_sft": {
"coverage": 3,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"highway is not present",
"giant rainbow flag is not present"
],
"rationale": "Correctly identifies a large waterfall but includes hallucinated elements (highway, rainbow flag) not in the image.",
"visual_specificity": 2
}
},
"city-traffic.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 1,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no elevators visible; main subject (heavy traffic) not mentioned"
],
"rationale": "Irrelevant focus on elevators and sidewalks; misses the prominent heavy traffic and tall buildings.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 2,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no parked cars (all cars are in traffic); no traffic lights visible; no bus present"
],
"rationale": "Falsely claims parked cars, traffic lights, and a bus; these elements are not visible in the image.",
"visual_specificity": 2
},
"muon_from_pretrain_pretrained": {
"coverage": 1,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"no bus visible in the image"
],
"rationale": "A tall building is present, but there is no bus. The main subject is heavy traffic, not a bus traveling to a building.",
"visual_specificity": 1
},
"muon_from_pretrain_sft": {
"coverage": 4,
"grounding_accuracy": 3,
"hallucination_control": 2,
"material_errors": [
"no motorcycles visible; no passersby (pedestrians) visible"
],
"rationale": "Correctly identifies busy street, traffic, tall buildings, but falsely includes motorcycles and passersby.",
"visual_specificity": 3
},
"muon_from_sft_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"mentions 'monitoring' which is not visible; traffic lights are not the main subject and not clearly present"
],
"rationale": "The image depicts a busy nighttime city street with traffic and buildings, not monitoring of traffic lights. No monitoring activity or distinct traffic lights are visible.",
"visual_specificity": 0
},
"muon_from_sft_sft": {
"coverage": 5,
"grounding_accuracy": 5,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Accurately describes the nighttime busy city street with cars, trucks, high-rise buildings, and bustling traffic (stationary and moving) without hallucinations.",
"visual_specificity": 4
},
"without_muon_pretrained": {
"coverage": 1,
"grounding_accuracy": 2,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Generic mention of 'city streets at nightfall' is accurate but lacks specific details about traffic, buildings, or activity.",
"visual_specificity": 0
},
"without_muon_sft": {
"coverage": 4,
"grounding_accuracy": 3,
"hallucination_control": 2,
"material_errors": [
"no pedestrians visible; no traffic lights clearly visible"
],
"rationale": "Correctly identifies busy street, traffic, cars, truck, tall buildings, and streetlights, but falsely mentions pedestrians and traffic lights which are not present.",
"visual_specificity": 3
}
},
"dance.jpg": {
"muon_from_dpo_pretrained": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 1,
"material_errors": [
"actors (image shows a dancer, not actors)"
],
"rationale": "Incorrectly identifies the subject as 'actors' instead of a dancer; no other accurate details.",
"visual_specificity": 0
},
"muon_from_dpo_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"black dress (actual: light-colored dress)",
"several people watching (no people present)",
"mobile phones (no phones in image)"
],
"rationale": "Incorrectly describes the dress as black (it is light-colored) and invents non-existent people and mobile phones.",
"visual_specificity": 0
},
"muon_from_pretrain_pretrained": {
"coverage": 1,
"grounding_accuracy": 1,
"hallucination_control": 2,
"material_errors": [
"performers (only one dancer)",
"colorful costumes (dress is light-colored, not colorful)"
],
"rationale": "Vaguely mentions a performance but incorrectly refers to multiple 'performers' and 'colorful costumes' (image has one dancer in a light dress).",
"visual_specificity": 0
},
"muon_from_pretrain_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"walking down walkway (dancing on stage)",
"holding umbrella (no umbrella)",
"chairs along walkway (no chairs)",
"kites (no kites)"
],
"rationale": "Contains multiple hallucinations: walkway, umbrella, chairs, and kites are all absent; the subject is dancing on stage, not walking.",
"visual_specificity": 0
},
"muon_from_sft_pretrained": {
"coverage": 4,
"grounding_accuracy": 5,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Accurately identifies a dancer performing on stage in a stylish outfit, with no false claims; aligns with visible elements (single dancer, stage, elegant dress).",
"visual_specificity": 3
},
"muon_from_sft_sft": {
"coverage": 0,
"grounding_accuracy": 0,
"hallucination_control": 0,
"material_errors": [
"tuxedo (wearing a dress)",
"holding microphone (no microphone)",
"several people around (no people)",
"wine glass and bottles (no such items)"
],
"rationale": "Contains multiple false claims: tuxedo (dress), microphone, people, and wine glasses/bottles are all absent.",
"visual_specificity": 0
},
"without_muon_pretrained": {
"coverage": 3,
"grounding_accuracy": 5,
"hallucination_control": 5,
"material_errors": [],
"rationale": "Correctly states a dancer is performing on stage but lacks visual details (e.g., outfit description).",
"visual_specificity": 1
},
"without_muon_sft": {
"coverage": 2,
"grounding_accuracy": 2,
"hallucination_control": 1,
"material_errors": [
"dance steps soaring high (dancer is on stage floor)",
"several chairs on stage (no chairs)",
"clock on stage (no clock)"
],
"rationale": "Mentions a dance on stage but invents 'soaring steps' (dancer is grounded) and non-existent chairs/clock.",
"visual_specificity": 1
}
}
},
"ranking_by_overall": [
"without_muon_sft",
"without_muon_pretrained",
"muon_from_dpo_sft",
"muon_from_sft_pretrained",
"muon_from_sft_sft",
"muon_from_pretrain_sft",
"muon_from_dpo_pretrained",
"muon_from_pretrain_pretrained"
],
"retained": {
"cells": 8,
"images": 8,
"outputs": 64
},
"schema_version": "exp8-4-summary-v1",
"scientific_findings": {
"author_claims_are_historical_observations_not_acceptance_gates": true,
"muon_only_causal_claim_avoided": true,
"sft_minus_pretrained_average": 0.1718,
"top_configuration": "without_muon_sft",
"top_configuration_overall": 1.9062
},
"stage_averages": {
"pretrained": {
"coverage": 1.125,
"grounding_accuracy": 1.375,
"hallucination_control": 1.875,
"overall": 1.2656,
"visual_specificity": 0.6875
},
"sft": {
"coverage": 1.7812,
"grounding_accuracy": 1.4688,
"hallucination_control": 1.0312,
"overall": 1.4374,
"visual_specificity": 1.4688
}
},
"status": "passed"
}