ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -0,0 +1,67 @@
|
||||
[
|
||||
{
|
||||
"purpose": "external-answer-judge",
|
||||
"provider": "moonshot",
|
||||
"endpoint": "https://api.moonshot.cn/v1",
|
||||
"started_at": "2026-07-29T18:54:20.300357+00:00",
|
||||
"latency_ms": 13009.478,
|
||||
"request": {
|
||||
"model": "moonshot-v1-8k",
|
||||
"seed": 37,
|
||||
"temperature": 0,
|
||||
"response_format": {
|
||||
"type": "json_object"
|
||||
},
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Independently judge chart QA answers. Return JSON {items:[{id,correct,score,reason}]}. Score 1 only if every requested quarter/value/difference matches the reference; otherwise 0."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "[{\"id\": \"png:native:highest\", \"question\": \"Which quarter had the highest revenue, and what was the exact value?\", \"reference\": \"Q4, $180M\", \"answer\": \"The highest revenue was in Q4, with an exact value of $180M. Visual evidence: The Q4 bar is the tallest in the chart, explicitly labeled \\\"$180M\\\" and exceeding the revenue of Q1 ($120M), Q2 ($150M), and Q3 ($95M).\\n\\n**Answer:** Q4 had the highest revenue at $180M.\"}, {\"id\": \"png:extract:highest\", \"question\": \"Which quarter had the highest revenue, and what was the exact value?\", \"reference\": \"Q4, $180M\", \"answer\": \"Q1 had the highest revenue, with an exact value of $180M.\"}, {\"id\": \"png:tool:highest\", \"question\": \"Which quarter had the highest revenue, and what was the exact value?\", \"reference\": \"Q4, $180M\", \"answer\": \"The highest revenue was in Q4 with an exact value of $180M.\"}, {\"id\": \"png:native:lowest_gap\", \"question\": \"Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?\", \"reference\": \"Q3, $95M; Q4 exceeded it by $85M\", \"answer\": \"- **Lowest revenue quarter**: Q3 \\n- **Value**: $95M \\n- **Q4 exceedance**: Q4 revenue ($180M) minus Q3 revenue ($95M) = $85M \\n\\n\\n**Visual evidence**: The red bar for Q3 is the shortest, labeled \\\"$95M,\\\" confirming it as the lowest. Q4’s purple bar is labeled \\\"$180M,\\\" the tallest. The vertical difference between Q4 and Q3 bars corresponds to $85M.\"}, {\"id\": \"png:extract:lowest_gap\", \"question\": \"Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?\", \"reference\": \"Q3, $95M; Q4 exceeded it by $85M\", \"answer\": \"The lowest revenue was in Q1 with $95M, and Q4 exceeded it by $85M.\"}, {\"id\": \"png:tool:lowest_gap\", \"question\": \"Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?\", \"reference\": \"Q3, $95M; Q4 exceeded it by $85M\", \"answer\": \"The lowest revenue occurred in Q3, with a value of $95M. Q4's revenue was $180M, exceeding Q3 by $85M ($180M - $95M). \\n\\n**Answers:** \\n- Lowest revenue quarter: Q3 \\n- Value: $95M \\n- Q4 exceeded it by: $85M\"}, {\"id\": \"pdf:native:highest\", \"question\": \"Which quarter had the highest revenue, and what was the exact value?\", \"reference\": \"Q4, $180M\", \"answer\": \"The fourth quarter (Q4) had the highest revenue, with an exact value of $180M. \\n\\n**Visual evidence**: The Q4 bar in the \\\"Acme Corp Quarterly Revenue 2024\\\" bar graph is the tallest, explicitly labeled \\\"$180M,\\\" exceeding Q1 ($120M), Q2 ($150M), and Q3 ($95M).\"}, {\"id\": \"pdf:extract:highest\", \"question\": \"Which quarter had the highest revenue, and what was the exact value?\", \"reference\": \"Q4, $180M\", \"answer\": \"The fourth quarter had the highest revenue, but the exact value is unavailable.\"}, {\"id\": \"pdf:tool:highest\", \"question\": \"Which quarter had the highest revenue, and what was the exact value?\", \"reference\": \"Q4, $180M\", \"answer\": \"The highest revenue in 2024 was in the fourth quarter (Q4), with an exact value of $180M. This is supported by the visual bar chart labeled \\\"Acme Corp Quarterly Revenue 2024,\\\" which shows Q4 as the tallest bar explicitly marked with \\\"$180M,\\\" exceeding Q1 ($120M), Q2 ($150M), and Q3 ($95M). The report also notes a \\\"strong recovery in the final quarter\\\" attributed to the launch of the new enterprise product line, confirming Q4's surge.\"}, {\"id\": \"pdf:native:lowest_gap\", \"question\": \"Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?\", \"reference\": \"Q3, $95M; Q4 exceeded it by $85M\", \"answer\": \"- **Lowest revenue quarter**: Q3 \\n- **Value**: $95M \\n- **Q4 exceedance**: Q4 revenue ($180M) minus Q3 revenue ($95M) = $85M \\n\\n\\n**Visual evidence**: The bar graph shows Q3 with the shortest bar, labeled $95M. Q4’s bar is the tallest, labeled $180M. The vertical gap between Q3 and Q4 bars corresponds to $85M.\"}, {\"id\": \"pdf:extract:lowest_gap\", \"question\": \"Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?\", \"reference\": \"Q3, $95M; Q4 exceeded it by $85M\", \"answer\": \"unavailable\"}, {\"id\": \"pdf:tool:lowest_gap\", \"question\": \"Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?\", \"reference\": \"Q3, $95M; Q4 exceeded it by $85M\", \"answer\": \"The lowest revenue quarter was Q3, with a value of $95M. Q4 revenue was $180M, exceeding Q3 by $85M. \\n\\n**Details**: The visual inspection confirms Q3 (red bar) as the lowest with $95M, while Q4 (purple bar) is the highest at $180M. The difference is calculated as $180M - $95M = $85M.\"}]"
|
||||
}
|
||||
]
|
||||
},
|
||||
"response": {
|
||||
"id": "chatcmpl-6a6a4c5c0efb1450ac9b2119",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"message": {
|
||||
"content": "{\"items\":[{\"id\": \"png:native:highest\", \"correct\": true, \"score\": 1, \"reason\": \"The answer correctly identifies Q4 as the quarter with the highest revenue and provides the exact value of $180M, matching the reference answer.\"}, {\"id\": \"png:extract:highest\", \"correct\": false, \"score\": 0, \"reason\": \"The answer incorrectly states Q1 as the quarter with the highest revenue instead of Q4.\"}, {\"id\": \"png:tool:highest\", \"correct\": true, \"score\": 1, \"reason\": \"The answer correctly identifies Q4 as the quarter with the highest revenue and provides the exact value of $180M, matching the reference answer.\"}, {\"id\": \"png:native:lowest_gap\", \"correct\": true, \"score\": 1, \"reason\": \"The answer correctly identifies Q3 as the quarter with the lowest revenue, provides the exact value of $95M, and correctly calculates the difference of $85M that Q4 exceeded it by, matching the reference answer.\"}, {\"id\": \"png:extract:lowest_gap\", \"correct\": false, \"score\": 0, \"reason\": \"The answer incorrectly states Q1 as the quarter with the lowest revenue instead of Q3.\"}, {\"id\": \"png:tool:lowest_gap\", \"correct\": true, \"score\": 1, \"reason\": \"The answer correctly identifies Q3 as the quarter with the lowest revenue, provides the exact value of $95M, and correctly calculates the difference of $85M that Q4 exceeded it by, matching the reference answer.\"}, {\"id\": \"pdf:native:highest\", \"correct\": true, \"score\": 1, \"reason\": \"The answer correctly identifies Q4 as the quarter with the highest revenue and provides the exact value of $180M, matching the reference answer.\"}, {\"id\": \"pdf:extract:highest\", \"correct\": false, \"score\": 0, \"reason\": \"The answer states that the exact value is unavailable, which contradicts the reference answer that provides the exact value of $180M.\"}, {\"id\": \"pdf:tool:highest\", \"correct\": true, \"score\": 1, \"reason\": \"The answer correctly identifies Q4 as the quarter with the highest revenue and provides the exact value of $180M, matching the reference answer and providing additional context.\"}, {\"id\": \"pdf:native:lowest_gap\", \"correct\": true, \"score\": 1, \"reason\": \"The answer correctly identifies Q3 as the quarter with the lowest revenue, provides the exact value of $95M, and correctly calculates the difference of $85M that Q4 exceeded it by, matching the reference answer.\"}, {\"id\": \"pdf:extract:lowest_gap\", \"correct\": false, \"score\": 0, \"reason\": \"The answer states that the information is unavailable, which contradicts the reference answer that provides the exact values and the difference.\"}, {\"id\": \"pdf:tool:lowest_gap\", \"correct\": true, \"score\": 1, \"reason\": \"The answer correctly identifies Q3 as the quarter with the lowest revenue, provides the exact value of $95M, and correctly calculates the difference of $85M that Q4 exceeded it by, matching the reference answer.\"}]}",
|
||||
"refusal": null,
|
||||
"role": "assistant",
|
||||
"annotations": null,
|
||||
"audio": null,
|
||||
"function_call": null,
|
||||
"tool_calls": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"created": 1785351261,
|
||||
"model": "moonshot-v1-8k",
|
||||
"object": "chat.completion",
|
||||
"service_tier": null,
|
||||
"system_fingerprint": null,
|
||||
"usage": {
|
||||
"completion_tokens": 781,
|
||||
"prompt_tokens": 1583,
|
||||
"total_tokens": 2364,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"completion_tokens": 781,
|
||||
"prompt_tokens": 1583,
|
||||
"total_tokens": 2364,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null
|
||||
},
|
||||
"response_model": "moonshot-v1-8k",
|
||||
"response_id": "chatcmpl-6a6a4c5c0efb1450ac9b2119"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "4-2",
|
||||
"run_id": "20260729T185433Z-4_2-e028c9db",
|
||||
"created_at": "2026-07-29T18:54:33.367744+00:00",
|
||||
"status": "passed",
|
||||
"run_dir": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/validation/runs/20260729T185433Z-4_2-e028c9db",
|
||||
"artifacts": {
|
||||
"evidence.json": "024358e7b58266eedbedda3cc880a61462c119a89c58e475f4c007dbcb8ff144",
|
||||
"receipts.json": "2aa961a1f7527417dd0d25ac6e542aaad9dc2ffaa7b40a4a450cb5fffe86ceac",
|
||||
"manifest.json": "1a9cc7bfd48717e73a03ebbde7fd786c7da2811a15267715a3794c0f1220362e"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/campaign.py",
|
||||
"sha256": "6f8162a0e124cef1acada4fbfadc6a1427aa487be24670aaceba2d668820ee3c",
|
||||
"bytes": 16104
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/create_sample.py",
|
||||
"sha256": "1f4fcd0e119a76a74c06744a585d31bc4bd4f755f813f8b2518f77127eaa1e72",
|
||||
"bytes": 4846
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/test_files/sample_chart.png",
|
||||
"sha256": "eaf8ac53fbc65a4e6ce0611d441cd12ddd2ed48ef8a727a9d6f22aef70605f39",
|
||||
"bytes": 38417
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/test_files/sample_report.pdf",
|
||||
"sha256": "db4828a20a12ad70dab5c196d23e4fa407f75c4e3cff2bda22c401de65cb9c2e",
|
||||
"bytes": 46368
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"native-multimodal": {
|
||||
"cases": 4,
|
||||
"exact_accuracy": 1.0,
|
||||
"judge_accuracy": 1.0,
|
||||
"mean_latency_ms": 8011.01425
|
||||
},
|
||||
"extract-to-text": {
|
||||
"cases": 4,
|
||||
"exact_accuracy": 0.0,
|
||||
"judge_accuracy": 0.0,
|
||||
"mean_latency_ms": 10867.427
|
||||
},
|
||||
"tool-on-demand": {
|
||||
"cases": 4,
|
||||
"exact_accuracy": 1.0,
|
||||
"judge_accuracy": 1.0,
|
||||
"mean_latency_ms": 29794.60875
|
||||
}
|
||||
},
|
||||
"acceptance": {
|
||||
"same_two_questions_all_paradigms_and_artifacts": true,
|
||||
"png_and_pdf_used": true,
|
||||
"chart_answers_absent_from_pdf_body_text": true,
|
||||
"real_native_vision_calls": true,
|
||||
"tool_selected_on_demand": true,
|
||||
"real_tool_vision_calls": true,
|
||||
"external_moonshot_judge": true,
|
||||
"all_calls_checkpointed": true
|
||||
}
|
||||
}
|
||||
+466
@@ -0,0 +1,466 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "4-2",
|
||||
"run_id": "20260729T185433Z-4_2-e028c9db",
|
||||
"provenance": {
|
||||
"captured_at": "2026-07-29T18:54:33.359939+00:00",
|
||||
"git_revision": "4a7f37cf278bd15948c409f14533017c4c7fbc29",
|
||||
"python": "3.11.4 (main, Jul 5 2023, 08:40:20) [Clang 14.0.6 ]",
|
||||
"platform": "macOS-26.3-arm64-arm-64bit",
|
||||
"credential_presence": {
|
||||
"ARK_API_KEY": true,
|
||||
"MOONSHOT_API_KEY": true,
|
||||
"OPENAI_API_KEY": true,
|
||||
"GEMINI_API_KEY": true,
|
||||
"SILICONFLOW_API_KEY": true
|
||||
}
|
||||
},
|
||||
"status": "passed",
|
||||
"providers": {
|
||||
"vision_answerer": {
|
||||
"provider": "Volcengine Ark",
|
||||
"endpoint": "https://ark.cn-beijing.volces.com/api/v3",
|
||||
"model": "doubao-seed-1-6-250615",
|
||||
"seed": 37
|
||||
},
|
||||
"judge": {
|
||||
"provider": "Moonshot",
|
||||
"endpoint": "https://api.moonshot.cn/v1",
|
||||
"model": "moonshot-v1-8k",
|
||||
"seed": 37
|
||||
}
|
||||
},
|
||||
"local_tools": {
|
||||
"tesseract": "tesseract 5.5.2",
|
||||
"pdftotext": "pdftotext version 25.06.0",
|
||||
"pdftoppm": "pdftoppm version 25.06.0"
|
||||
},
|
||||
"artifacts": [
|
||||
{
|
||||
"kind": "png",
|
||||
"source_path": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/test_files/sample_chart.png",
|
||||
"source_sha256": "eaf8ac53fbc65a4e6ce0611d441cd12ddd2ed48ef8a727a9d6f22aef70605f39",
|
||||
"visual_input": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/test_files/sample_chart.png",
|
||||
"visual_sha256": "eaf8ac53fbc65a4e6ce0611d441cd12ddd2ed48ef8a727a9d6f22aef70605f39",
|
||||
"extracted_text": "Acme Corp Quarterly Revenue 2024\n200\n$180M\n175\n$150M\n150\nB 5 $120M\n&\n8 100 $95M\nG\n>\n2p\n50\n25\n0\nQl Q2 Q3 a4",
|
||||
"extraction": {
|
||||
"command": [
|
||||
"tesseract",
|
||||
"/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/test_files/sample_chart.png",
|
||||
"stdout",
|
||||
"--psm",
|
||||
"6"
|
||||
],
|
||||
"stderr": "",
|
||||
"latency_ms": 203.405
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "pdf",
|
||||
"source_path": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/test_files/sample_report.pdf",
|
||||
"source_sha256": "db4828a20a12ad70dab5c196d23e4fa407f75c4e3cff2bda22c401de65cb9c2e",
|
||||
"visual_input": "/var/folders/0l/vk1w1b5n2fxfwdlz3f_w25_w0000gp/T/tmpqt5feern/sample_report_page.png",
|
||||
"visual_sha256": "5688daad3a127e783fbb42e71d1d00bd8077d36a1ccf53e19f323d59b4e1d1e5",
|
||||
"extracted_text": "Acme Corp 2024 Revenue Report\n\nThis internal report summarizes Acme Corp's revenue performance in 2024. Overall the year\nshowed healthy growth, with a mid-year dip followed by a strong recovery in the final quarter. The\nchart below breaks down revenue by quarter; management attributes the fourth-quarter surge to the\nlaunch of the new enterprise product line.",
|
||||
"extraction": {
|
||||
"command": [
|
||||
"pdftotext",
|
||||
"-layout",
|
||||
"/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/test_files/sample_report.pdf",
|
||||
"-"
|
||||
],
|
||||
"stderr": "",
|
||||
"latency_ms": 20.134
|
||||
}
|
||||
}
|
||||
],
|
||||
"questions": [
|
||||
{
|
||||
"id": "highest",
|
||||
"question": "Which quarter had the highest revenue, and what was the exact value?",
|
||||
"expected": "Q4, $180M",
|
||||
"required_patterns": [
|
||||
"\\bQ4\\b",
|
||||
"(?:\\$\\s*)?180\\s*M"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "lowest_gap",
|
||||
"question": "Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?",
|
||||
"expected": "Q3, $95M; Q4 exceeded it by $85M",
|
||||
"required_patterns": [
|
||||
"\\bQ3\\b",
|
||||
"(?:\\$\\s*)?95\\s*M",
|
||||
"(?:\\$\\s*)?85\\s*M"
|
||||
]
|
||||
}
|
||||
],
|
||||
"results": [
|
||||
{
|
||||
"artifact": "png",
|
||||
"question_id": "highest",
|
||||
"question": "Which quarter had the highest revenue, and what was the exact value?",
|
||||
"expected": "Q4, $180M",
|
||||
"id": "png:native:highest",
|
||||
"paradigm": "native-multimodal",
|
||||
"answer": "The highest revenue was in Q4, with an exact value of $180M. Visual evidence: The Q4 bar is the tallest in the chart, explicitly labeled \"$180M\" and exceeding the revenue of Q1 ($120M), Q2 ($150M), and Q3 ($95M).\n\n**Answer:** Q4 had the highest revenue at $180M.",
|
||||
"latency_ms": 8094.933,
|
||||
"exact_correct": true,
|
||||
"external_judge": {
|
||||
"id": "png:native:highest",
|
||||
"correct": true,
|
||||
"score": 1,
|
||||
"reason": "The answer correctly identifies Q4 as the quarter with the highest revenue and provides the exact value of $180M, matching the reference answer."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "png",
|
||||
"question_id": "highest",
|
||||
"question": "Which quarter had the highest revenue, and what was the exact value?",
|
||||
"expected": "Q4, $180M",
|
||||
"id": "png:extract:highest",
|
||||
"paradigm": "extract-to-text",
|
||||
"answer": "Q1 had the highest revenue, with an exact value of $180M.",
|
||||
"latency_ms": 7104.31,
|
||||
"exact_correct": false,
|
||||
"external_judge": {
|
||||
"id": "png:extract:highest",
|
||||
"correct": false,
|
||||
"score": 0,
|
||||
"reason": "The answer incorrectly states Q1 as the quarter with the highest revenue instead of Q4."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "png",
|
||||
"question_id": "highest",
|
||||
"question": "Which quarter had the highest revenue, and what was the exact value?",
|
||||
"expected": "Q4, $180M",
|
||||
"id": "png:tool:highest",
|
||||
"paradigm": "tool-on-demand",
|
||||
"answer": "The highest revenue was in Q4 with an exact value of $180M.",
|
||||
"latency_ms": 31752.022,
|
||||
"exact_correct": true,
|
||||
"tool_trace": {
|
||||
"tool_selected": true,
|
||||
"decision": {
|
||||
"content": "",
|
||||
"refusal": null,
|
||||
"role": "assistant",
|
||||
"annotations": null,
|
||||
"audio": null,
|
||||
"function_call": null,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_ep2ovjm4i87gakzdym8c7scs",
|
||||
"function": {
|
||||
"arguments": "{\"question\":\"What is the revenue for each quarter (Q1, Q2, Q3, Q4) in the Acme Corp Quarterly Revenue 2024 chart, and which quarter has the highest revenue with its exact value?\"}",
|
||||
"name": "inspect_visual"
|
||||
},
|
||||
"type": "function"
|
||||
}
|
||||
],
|
||||
"reasoning_content": "我现在需要解决的问题是：“Which quarter had the highest revenue, and what was the exact value?” 也就是哪个季度的收入最高，具体数值是多少。首先，我得仔细看看用户提供的提取文本，看看有没有直接的信息。\n\n提取的文本是“Acme Corp Quarterly Revenue 2024”，然后下面有一些数字和符号，看起来像是图表的数据。具体的数字有200、180M、175、150M、150、B 5 $120M、&、8 100 $95M、G、>、2p、50、25、0，还有Ql Q2 Q3 a4。这里的Ql应该是Q1，a4可能是Q4的笔误。不过这些数字看起来比较混乱，可能是OCR识别错误或者格式问题。\n\n首先，我需要确定每个季度对应的收入数值。文本中出现了几个带有“M”的数值，比如$180M、$150M、$120M、$95M，这些可能是百万美元的收入。但问题在于这些数值和季度的对应关系不明确。比如，Q1、Q2、Q3、Q4分别对应哪个数值呢？\n\n提取的文本里有“Ql Q2 Q3 a4”，可能是Q1到Q4的顺序。但前面的数字排列看起来没有明显的顺序。比如200、175、150、100、50、25、0这些可能是纵轴的刻度，但具体对应哪个季度的数值呢？比如$180M可能对应某个季度，但文本中没有明确说明每个季度的具体数值。\n\n例如，文本中有“$180M”，但它旁边的数字是200和175，可能200是刻度，而180M是实际数值？或者175对应的是$150M？这里的空间关联不明确，因为文本提取可能没有正确捕捉到图表中的位置关系。比如，在柱状图或折线图中，每个季度的柱子或点会对应一个数值，但文本提取可能把这些数值和刻度混在一起了。\n\n因此，根据问题要求，当文本中没有明确建立精确的图表数值或空间关联时，需要调用inspect_visual工具。这里显然各个季度的收入数值没有明确对应，比如Q1的收入是多少，Q2是多少等，文本中的数字排列混乱，无法直接确定。所以必须通过视觉检查来确认每个季度对应的具体收入数值，从而找出最高的那个季度及其数值。\n\n所以，我需要调用inspect_visual工具，询问每个季度的收入数值，特别是最高的那个。具体的问题应该是：“What is the revenue for each quarter (Q1, Q2, Q3, Q4) in the Acme Corp Quarterly Revenue 2024 chart, and which quarter has the highest revenue with its exact value?” 这样就能明确获取所需的信息了。"
|
||||
},
|
||||
"executions": [
|
||||
{
|
||||
"tool_call_id": "call_ep2ovjm4i87gakzdym8c7scs",
|
||||
"name": "inspect_visual",
|
||||
"arguments": {
|
||||
"question": "What is the revenue for each quarter (Q1, Q2, Q3, Q4) in the Acme Corp Quarterly Revenue 2024 chart, and which quarter has the highest revenue with its exact value?"
|
||||
},
|
||||
"result": "**Quarterly Revenues:** \n- Q1: $120M (blue bar, labeled directly on the bar) \n- Q2: $150M (green bar, labeled directly on the bar) \n- Q3: $95M (red bar, labeled directly on the bar) \n- Q4: $180M (purple bar, labeled directly on the bar) \n\n\n**Highest Revenue:** Q4 with $180M (the tallest bar, explicitly labeled $180M)."
|
||||
}
|
||||
]
|
||||
},
|
||||
"external_judge": {
|
||||
"id": "png:tool:highest",
|
||||
"correct": true,
|
||||
"score": 1,
|
||||
"reason": "The answer correctly identifies Q4 as the quarter with the highest revenue and provides the exact value of $180M, matching the reference answer."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "png",
|
||||
"question_id": "lowest_gap",
|
||||
"question": "Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?",
|
||||
"expected": "Q3, $95M; Q4 exceeded it by $85M",
|
||||
"id": "png:native:lowest_gap",
|
||||
"paradigm": "native-multimodal",
|
||||
"answer": "- **Lowest revenue quarter**: Q3 \n- **Value**: $95M \n- **Q4 exceedance**: Q4 revenue ($180M) minus Q3 revenue ($95M) = $85M \n\n\n**Visual evidence**: The red bar for Q3 is the shortest, labeled \"$95M,\" confirming it as the lowest. Q4’s purple bar is labeled \"$180M,\" the tallest. The vertical difference between Q4 and Q3 bars corresponds to $85M.",
|
||||
"latency_ms": 10174.016,
|
||||
"exact_correct": true,
|
||||
"external_judge": {
|
||||
"id": "png:native:lowest_gap",
|
||||
"correct": true,
|
||||
"score": 1,
|
||||
"reason": "The answer correctly identifies Q3 as the quarter with the lowest revenue, provides the exact value of $95M, and correctly calculates the difference of $85M that Q4 exceeded it by, matching the reference answer."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "png",
|
||||
"question_id": "lowest_gap",
|
||||
"question": "Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?",
|
||||
"expected": "Q3, $95M; Q4 exceeded it by $85M",
|
||||
"id": "png:extract:lowest_gap",
|
||||
"paradigm": "extract-to-text",
|
||||
"answer": "The lowest revenue was in Q1 with $95M, and Q4 exceeded it by $85M.",
|
||||
"latency_ms": 29615.343,
|
||||
"exact_correct": false,
|
||||
"external_judge": {
|
||||
"id": "png:extract:lowest_gap",
|
||||
"correct": false,
|
||||
"score": 0,
|
||||
"reason": "The answer incorrectly states Q1 as the quarter with the lowest revenue instead of Q3."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "png",
|
||||
"question_id": "lowest_gap",
|
||||
"question": "Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?",
|
||||
"expected": "Q3, $95M; Q4 exceeded it by $85M",
|
||||
"id": "png:tool:lowest_gap",
|
||||
"paradigm": "tool-on-demand",
|
||||
"answer": "The lowest revenue occurred in Q3, with a value of $95M. Q4's revenue was $180M, exceeding Q3 by $85M ($180M - $95M). \n\n**Answers:** \n- Lowest revenue quarter: Q3 \n- Value: $95M \n- Q4 exceeded it by: $85M",
|
||||
"latency_ms": 35775.984,
|
||||
"exact_correct": true,
|
||||
"tool_trace": {
|
||||
"tool_selected": true,
|
||||
"decision": {
|
||||
"content": "",
|
||||
"refusal": null,
|
||||
"role": "assistant",
|
||||
"annotations": null,
|
||||
"audio": null,
|
||||
"function_call": null,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_d6od6lf1thf1cpwv1v6vjzje",
|
||||
"function": {
|
||||
"arguments": "{\"question\":\"What are the revenue values for Q1, Q2, Q3, and Q4 of Acme Corp in 2024? Which quarter has the lowest revenue, what is its value, and what is Q4's revenue?\"}",
|
||||
"name": "inspect_visual"
|
||||
},
|
||||
"type": "function"
|
||||
}
|
||||
],
|
||||
"reasoning_content": "我现在需要解决的问题是：哪个季度的收入最低，其数值是多少，以及第四季度（Q4）比它高出多少？首先，我得仔细看看提供的提取文本，看看有没有直接的信息。\n\n提取的文本看起来有点混乱，可能是OCR识别的结果不太准确。首先，标题是“Acme Corp Quarterly Revenue 2024”，然后有一些数字和符号。比如“200”、“$180M”、“175”、“$150M”、“150”、“B 5 $120M”、“&”、“8 100 $95M”、“G”、“>”、“2p”、“50”、“25”、“0”，最后是“Ql Q2 Q3 a4”。这里的“Ql”应该是Q1，“a4”可能是Q4的笔误。\n\n首先，我需要确定每个季度对应的收入数值。通常季度是Q1到Q4，所以文本中的Ql、Q2、Q3、a4应该对应这四个季度。但问题在于，这些数字和符号如何与季度对应起来？比如，“$180M”、“$150M”、“$120M”、“$95M”看起来像是可能的收入数值，单位是百万美元。但这些数值是如何分配到各个季度的呢？\n\n提取的文本中没有明确说明每个季度对应的具体数值，只是列出了一些数字和符号。例如，“Ql Q2 Q3 a4”可能是在图表的底部，作为x轴的标签，而上面的数字可能是y轴的刻度或者数据点。但这里的文本提取似乎没有正确对应，比如“B 5 $120M”、“8 100 $95M”这样的组合，可能是识别错误，或者是图表中的某些元素被错误提取。\n\n在这种情况下，由于文本提取不明确，无法直接确定每个季度的收入数值，特别是空间关联（即哪个数值对应哪个季度）没有被文本明确建立。根据系统提示，当需要精确的图表数值或空间关联时，应该调用inspect_visual工具。\n\n因此，我需要调用inspect_visual工具，询问每个季度的具体收入数值，特别是最低的那个季度及其数值，以及Q4的数值，以便计算差额。具体来说，问题应该是：“请提供Acme Corp 2024年各季度（Q1、Q2、Q3、Q4）的收入数值，并指出最低收入的季度及其数值，以及Q4的收入数值。” 这样就能获取所需的精确数据，进而回答问题。"
|
||||
},
|
||||
"executions": [
|
||||
{
|
||||
"tool_call_id": "call_d6od6lf1thf1cpwv1v6vjzje",
|
||||
"name": "inspect_visual",
|
||||
"arguments": {
|
||||
"question": "What are the revenue values for Q1, Q2, Q3, and Q4 of Acme Corp in 2024? Which quarter has the lowest revenue, what is its value, and what is Q4's revenue?"
|
||||
},
|
||||
"result": "**Revenue Values by Quarter (2024):** \n- Q1: $120M (blue bar) \n- Q2: $150M (green bar) \n- Q3: $95M (red bar) \n- Q4: $180M (purple bar) \n\n\n**Lowest Revenue:** Q3, with $95M (smallest bar height, labeled directly). \n\n\n**Q4 Revenue:** $180M (tallest bar, labeled directly). \n\n\n*Visual Evidence:* The bar chart explicitly labels each quarter’s revenue above its respective bar, confirming the values. Q3’s red bar is the shortest, and Q4’s purple bar is the tallest."
|
||||
}
|
||||
]
|
||||
},
|
||||
"external_judge": {
|
||||
"id": "png:tool:lowest_gap",
|
||||
"correct": true,
|
||||
"score": 1,
|
||||
"reason": "The answer correctly identifies Q3 as the quarter with the lowest revenue, provides the exact value of $95M, and correctly calculates the difference of $85M that Q4 exceeded it by, matching the reference answer."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "pdf",
|
||||
"question_id": "highest",
|
||||
"question": "Which quarter had the highest revenue, and what was the exact value?",
|
||||
"expected": "Q4, $180M",
|
||||
"id": "pdf:native:highest",
|
||||
"paradigm": "native-multimodal",
|
||||
"answer": "The fourth quarter (Q4) had the highest revenue, with an exact value of $180M. \n\n**Visual evidence**: The Q4 bar in the \"Acme Corp Quarterly Revenue 2024\" bar graph is the tallest, explicitly labeled \"$180M,\" exceeding Q1 ($120M), Q2 ($150M), and Q3 ($95M).",
|
||||
"latency_ms": 6289.762,
|
||||
"exact_correct": true,
|
||||
"external_judge": {
|
||||
"id": "pdf:native:highest",
|
||||
"correct": true,
|
||||
"score": 1,
|
||||
"reason": "The answer correctly identifies Q4 as the quarter with the highest revenue and provides the exact value of $180M, matching the reference answer."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "pdf",
|
||||
"question_id": "highest",
|
||||
"question": "Which quarter had the highest revenue, and what was the exact value?",
|
||||
"expected": "Q4, $180M",
|
||||
"id": "pdf:extract:highest",
|
||||
"paradigm": "extract-to-text",
|
||||
"answer": "The fourth quarter had the highest revenue, but the exact value is unavailable.",
|
||||
"latency_ms": 3313.326,
|
||||
"exact_correct": false,
|
||||
"external_judge": {
|
||||
"id": "pdf:extract:highest",
|
||||
"correct": false,
|
||||
"score": 0,
|
||||
"reason": "The answer states that the exact value is unavailable, which contradicts the reference answer that provides the exact value of $180M."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "pdf",
|
||||
"question_id": "highest",
|
||||
"question": "Which quarter had the highest revenue, and what was the exact value?",
|
||||
"expected": "Q4, $180M",
|
||||
"id": "pdf:tool:highest",
|
||||
"paradigm": "tool-on-demand",
|
||||
"answer": "The highest revenue in 2024 was in the fourth quarter (Q4), with an exact value of $180M. This is supported by the visual bar chart labeled \"Acme Corp Quarterly Revenue 2024,\" which shows Q4 as the tallest bar explicitly marked with \"$180M,\" exceeding Q1 ($120M), Q2 ($150M), and Q3 ($95M). The report also notes a \"strong recovery in the final quarter\" attributed to the launch of the new enterprise product line, confirming Q4's surge.",
|
||||
"latency_ms": 23111.276,
|
||||
"exact_correct": true,
|
||||
"tool_trace": {
|
||||
"tool_selected": true,
|
||||
"decision": {
|
||||
"content": "",
|
||||
"refusal": null,
|
||||
"role": "assistant",
|
||||
"annotations": null,
|
||||
"audio": null,
|
||||
"function_call": null,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_m638i2ydhoymnhz26ol5oad4",
|
||||
"function": {
|
||||
"arguments": "{\"question\":\"Which quarter had the highest revenue in 2024, and what was the exact revenue value for that quarter?\"}",
|
||||
"name": "inspect_visual"
|
||||
},
|
||||
"type": "function"
|
||||
}
|
||||
],
|
||||
"reasoning_content": "我现在需要回答用户的问题：“Which quarter had the highest revenue, and what was the exact value?” 根据提供的提取文本，里面提到2024年整体表现健康增长，年中有所下滑，然后第四季度强劲复苏，管理层将第四季度的增长归因于新企业产品线的推出。不过，文本中并没有给出具体的季度收入数值，也没有明确说明哪个季度的收入最高，只是提到第四季度有增长。\n\n根据系统提示，当文本中没有明确的图表数值或空间关联时，需要调用inspect_visual工具。这里的问题需要知道具体的季度收入数值以及哪个季度最高，而文本中只提到第四季度有增长，但没有具体数字，所以必须通过视觉检查来获取这些信息。因此，我需要调用inspect_visual工具，询问哪个季度收入最高以及具体数值。"
|
||||
},
|
||||
"executions": [
|
||||
{
|
||||
"tool_call_id": "call_m638i2ydhoymnhz26ol5oad4",
|
||||
"name": "inspect_visual",
|
||||
"arguments": {
|
||||
"question": "Which quarter had the highest revenue in 2024, and what was the exact revenue value for that quarter?"
|
||||
},
|
||||
"result": "The highest revenue in 2024 was in Q4, with an exact revenue value of $180M. \n\n**Supporting visual evidence**: The bar chart labeled \"Acme Corp Quarterly Revenue 2024\" shows Q4 as the tallest bar, explicitly marked with \"$180M.\" This exceeds Q1 ($120M), Q2 ($150M), and Q3 ($95M). The report also notes a \"strong recovery in the final quarter\" attributed to the new enterprise product line, confirming Q4's surge."
|
||||
}
|
||||
]
|
||||
},
|
||||
"external_judge": {
|
||||
"id": "pdf:tool:highest",
|
||||
"correct": true,
|
||||
"score": 1,
|
||||
"reason": "The answer correctly identifies Q4 as the quarter with the highest revenue and provides the exact value of $180M, matching the reference answer and providing additional context."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "pdf",
|
||||
"question_id": "lowest_gap",
|
||||
"question": "Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?",
|
||||
"expected": "Q3, $95M; Q4 exceeded it by $85M",
|
||||
"id": "pdf:native:lowest_gap",
|
||||
"paradigm": "native-multimodal",
|
||||
"answer": "- **Lowest revenue quarter**: Q3 \n- **Value**: $95M \n- **Q4 exceedance**: Q4 revenue ($180M) minus Q3 revenue ($95M) = $85M \n\n\n**Visual evidence**: The bar graph shows Q3 with the shortest bar, labeled $95M. Q4’s bar is the tallest, labeled $180M. The vertical gap between Q3 and Q4 bars corresponds to $85M.",
|
||||
"latency_ms": 7485.346,
|
||||
"exact_correct": true,
|
||||
"external_judge": {
|
||||
"id": "pdf:native:lowest_gap",
|
||||
"correct": true,
|
||||
"score": 1,
|
||||
"reason": "The answer correctly identifies Q3 as the quarter with the lowest revenue, provides the exact value of $95M, and correctly calculates the difference of $85M that Q4 exceeded it by, matching the reference answer."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "pdf",
|
||||
"question_id": "lowest_gap",
|
||||
"question": "Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?",
|
||||
"expected": "Q3, $95M; Q4 exceeded it by $85M",
|
||||
"id": "pdf:extract:lowest_gap",
|
||||
"paradigm": "extract-to-text",
|
||||
"answer": "unavailable",
|
||||
"latency_ms": 3436.729,
|
||||
"exact_correct": false,
|
||||
"external_judge": {
|
||||
"id": "pdf:extract:lowest_gap",
|
||||
"correct": false,
|
||||
"score": 0,
|
||||
"reason": "The answer states that the information is unavailable, which contradicts the reference answer that provides the exact values and the difference."
|
||||
}
|
||||
},
|
||||
{
|
||||
"artifact": "pdf",
|
||||
"question_id": "lowest_gap",
|
||||
"question": "Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?",
|
||||
"expected": "Q3, $95M; Q4 exceeded it by $85M",
|
||||
"id": "pdf:tool:lowest_gap",
|
||||
"paradigm": "tool-on-demand",
|
||||
"answer": "The lowest revenue quarter was Q3, with a value of $95M. Q4 revenue was $180M, exceeding Q3 by $85M. \n\n**Details**: The visual inspection confirms Q3 (red bar) as the lowest with $95M, while Q4 (purple bar) is the highest at $180M. The difference is calculated as $180M - $95M = $85M.",
|
||||
"latency_ms": 28539.153,
|
||||
"exact_correct": true,
|
||||
"tool_trace": {
|
||||
"tool_selected": true,
|
||||
"decision": {
|
||||
"content": "",
|
||||
"refusal": null,
|
||||
"role": "assistant",
|
||||
"annotations": null,
|
||||
"audio": null,
|
||||
"function_call": null,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_df307c9zp27ddcazuecubeqh",
|
||||
"function": {
|
||||
"arguments": "{\"question\":\"Which quarter had the lowest revenue in 2024, what was its revenue value, what was the revenue value of Q4, and by how much did Q4 exceed the lowest quarter?\"}",
|
||||
"name": "inspect_visual"
|
||||
},
|
||||
"type": "function"
|
||||
}
|
||||
],
|
||||
"reasoning_content": "我现在需要回答用户的问题：“Which quarter had the lowest revenue, what was its value, and by how much did Q4 exceed it?” 首先，我得仔细看看提供的提取文本，看看有没有相关的信息。\n\n根据提取的文本，里面提到“Overall the year showed healthy growth, with a mid-year dip followed by a strong recovery in the final quarter.” 这里提到了年中出现了一个低谷，然后第四季度强劲复苏。所以，最低的季度应该是年中的某个季度，可能是第二季度或者第三季度？不过文本里没有明确说是哪个季度，只是说“mid-year dip”，也就是年中低谷。年中通常指的是第二季度或者第三季度，但具体哪个呢？这时候可能需要视觉检查工具来确认图表中的具体数据。\n\n接下来，问题还问最低收入的具体数值，以及第四季度超过它多少。文本中没有提到任何具体的数值，只提到了第四季度的增长是由于新产品线的推出。所以，这些具体的数值必须通过查看图表才能获得。因此，我需要调用inspect_visual工具来获取这些信息。\n\n所以，我需要分步骤来处理：首先确定哪个季度是最低的，然后获取该季度的数值，再获取第四季度的数值，最后计算两者之间的差值。但根据文本，只能推断出年中低谷，具体哪个季度和数值都不知道，所以必须通过视觉检查来确认。\n\n因此，我应该调用inspect_visual工具，提出以下问题：“Which quarter had the lowest revenue in 2024, what was its revenue value, and what was the revenue value of Q4? Also, calculate the difference between Q4 and the lowest quarter.” 这样就能一次性获取所有需要的信息，包括最低季度、其数值、Q4的数值以及两者的差值。\n\n不过，可能需要更明确的问题结构。比如，先问最低的季度，再问该季度的数值，然后问Q4的数值，最后计算差值。但工具可能可以一次处理多个问题，所以综合成一个问题应该可以。或者分开调用？不过用户允许并行调用，所以可能可以一次问多个问题。不过根据工具的描述，每次调用只能有一个question参数，所以可能需要分多次调用？或者在一个question里包含多个子问题？\n\n根据工具的parameters，question是一个字符串，所以可能可以在一个问题里包含多个询问点。例如：“Which quarter had the lowest revenue in 2024? What was its revenue value? What was the revenue value of Q4? By how much did Q4 exceed the lowest quarter?” 这样工具可能会返回所有这些信息。\n\n因此，我应该构造一个包含所有这些问题的查询，调用inspect_visual工具，以获取所需的数据。这样就能回答用户的问题了。"
|
||||
},
|
||||
"executions": [
|
||||
{
|
||||
"tool_call_id": "call_df307c9zp27ddcazuecubeqh",
|
||||
"name": "inspect_visual",
|
||||
"arguments": {
|
||||
"question": "Which quarter had the lowest revenue in 2024, what was its revenue value, what was the revenue value of Q4, and by how much did Q4 exceed the lowest quarter?"
|
||||
},
|
||||
"result": "- **Lowest revenue quarter**: Q3 \n- **Lowest revenue value**: $95M \n- **Q4 revenue value**: $180M \n- **Q4 exceeds lowest quarter by**: $85M \n\n\n**Visual evidence**: The bar chart shows Q3 (red bar) with the shortest height, labeled $95M. Q4 (purple bar) is the tallest, labeled $180M. The difference is $180M - $95M = $85M."
|
||||
}
|
||||
]
|
||||
},
|
||||
"external_judge": {
|
||||
"id": "pdf:tool:lowest_gap",
|
||||
"correct": true,
|
||||
"score": 1,
|
||||
"reason": "The answer correctly identifies Q3 as the quarter with the lowest revenue, provides the exact value of $95M, and correctly calculates the difference of $85M that Q4 exceeded it by, matching the reference answer."
|
||||
}
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"native-multimodal": {
|
||||
"cases": 4,
|
||||
"exact_accuracy": 1.0,
|
||||
"judge_accuracy": 1.0,
|
||||
"mean_latency_ms": 8011.01425
|
||||
},
|
||||
"extract-to-text": {
|
||||
"cases": 4,
|
||||
"exact_accuracy": 0.0,
|
||||
"judge_accuracy": 0.0,
|
||||
"mean_latency_ms": 10867.427
|
||||
},
|
||||
"tool-on-demand": {
|
||||
"cases": 4,
|
||||
"exact_accuracy": 1.0,
|
||||
"judge_accuracy": 1.0,
|
||||
"mean_latency_ms": 29794.60875
|
||||
}
|
||||
},
|
||||
"acceptance": {
|
||||
"same_two_questions_all_paradigms_and_artifacts": true,
|
||||
"png_and_pdf_used": true,
|
||||
"chart_answers_absent_from_pdf_body_text": true,
|
||||
"real_native_vision_calls": true,
|
||||
"tool_selected_on_demand": true,
|
||||
"real_tool_vision_calls": true,
|
||||
"external_moonshot_judge": true,
|
||||
"all_calls_checkpointed": true
|
||||
},
|
||||
"checkpoint_files": [
|
||||
"/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/validation/checkpoints/20260729T185104Z-ark.json",
|
||||
"/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/validation/checkpoints/20260729T185104Z-judge.json"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "4-2",
|
||||
"run_id": "20260729T185433Z-4_2-e028c9db",
|
||||
"created_at": "2026-07-29T18:54:33.367744+00:00",
|
||||
"status": "passed",
|
||||
"run_dir": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/validation/runs/20260729T185433Z-4_2-e028c9db",
|
||||
"artifacts": {
|
||||
"evidence.json": "024358e7b58266eedbedda3cc880a61462c119a89c58e475f4c007dbcb8ff144",
|
||||
"receipts.json": "2aa961a1f7527417dd0d25ac6e542aaad9dc2ffaa7b40a4a450cb5fffe86ceac"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/campaign.py",
|
||||
"sha256": "6f8162a0e124cef1acada4fbfadc6a1427aa487be24670aaceba2d668820ee3c",
|
||||
"bytes": 16104
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/create_sample.py",
|
||||
"sha256": "1f4fcd0e119a76a74c06744a585d31bc4bd4f755f813f8b2518f77127eaa1e72",
|
||||
"bytes": 4846
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/test_files/sample_chart.png",
|
||||
"sha256": "eaf8ac53fbc65a4e6ce0611d441cd12ddd2ed48ef8a727a9d6f22aef70605f39",
|
||||
"bytes": 38417
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter4/multimodal-agent/test_files/sample_report.pdf",
|
||||
"sha256": "db4828a20a12ad70dab5c196d23e4fa407f75c4e3cff2bda22c401de65cb9c2e",
|
||||
"bytes": 46368
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"native-multimodal": {
|
||||
"cases": 4,
|
||||
"exact_accuracy": 1.0,
|
||||
"judge_accuracy": 1.0,
|
||||
"mean_latency_ms": 8011.01425
|
||||
},
|
||||
"extract-to-text": {
|
||||
"cases": 4,
|
||||
"exact_accuracy": 0.0,
|
||||
"judge_accuracy": 0.0,
|
||||
"mean_latency_ms": 10867.427
|
||||
},
|
||||
"tool-on-demand": {
|
||||
"cases": 4,
|
||||
"exact_accuracy": 1.0,
|
||||
"judge_accuracy": 1.0,
|
||||
"mean_latency_ms": 29794.60875
|
||||
}
|
||||
},
|
||||
"acceptance": {
|
||||
"same_two_questions_all_paradigms_and_artifacts": true,
|
||||
"png_and_pdf_used": true,
|
||||
"chart_answers_absent_from_pdf_body_text": true,
|
||||
"real_native_vision_calls": true,
|
||||
"tool_selected_on_demand": true,
|
||||
"real_tool_vision_calls": true,
|
||||
"external_moonshot_judge": true,
|
||||
"all_calls_checkpointed": true
|
||||
}
|
||||
}
|
||||
+1987
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user