liqiang/ai-agent-book
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
Details
dependency resolution / resolve (3.11) (push) Canceled after 0s
Details
dependency resolution / resolve (3.13) (push) Canceled after 0s
Details
deploy-pages / build (push) Canceled after 0s
Details
deploy-pages / deploy (push) Canceled after 0s
Details
i18n consistency check / check (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
Details
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
Details
web-search-agent tests / test (push) Canceled after 0s
Details
web-search-agent tests / agentbook (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
liqiang
2026-08-20 13:12:50 +00:00
commit b119135836
10275 changed files with 3284984 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/comparison.json
+700
View File
@@ -0,0 +1,700 @@
{
"experiment_id": "2-4",
"created_at": "2026-07-30T02:21:25.575202+08:00",
"protocol_sha256": "ebedaecdbb6c4fc8d8d92222dc17334a27a1b1b9f19e4732578563f5e9389424",
"protocol_copy": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/experiment_protocol.json",
"provider": "openai",
"model": "gpt-4o-mini",
"user_model": "gpt-4o-mini",
"objective_scoring": "vendored tau-bench environment reward",
"arms": {
"baseline": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-baseline_0730022052.json",
"sha256": "0edf8d281ea5e5d39ca5079d445b1684b1ec8de1026013590d68c5a0e88121fd"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
},
"tone_trump": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_trump_0730022059.json",
"sha256": "1531727c765cd3b1caef4b3c486399467979a40f951c603ae577241ed7918ccb"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
},
"tone_casual": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_0730022104.json",
"sha256": "4a1205d5ee3394b18e95d184a2f6dc7a565a2c1ba1558428b83f394f3d989b51"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
},
"wiki_random": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-wiki_random_0730022109.json",
"sha256": "bca0c69faf8faceabf444cc5f85c51bd7b4db4b72ea83151ed1ea88e60891264"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
},
"no_tool_desc": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-no_tool_desc_0730022115.json",
"sha256": "7a0fd841f28b3c6a8fe0f801bb33fba08acbdaa3a3dadbc0330334d209592ebf"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
},
"all_ablations": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_wiki_random_no_tool_desc_0730022120.json",
"sha256": "aee798ceabad4e2022adcbadef6aca739937be4fbeffc9f98c9c65950bbd3990"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
}
},
"credential_scan_passed": true,
"credential_findings": [],
"usage_and_cost": {
"total_real_api_calls": 0,
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0,
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"qualification": "provider usage with LiteLLM response-cost estimate"
},
"campaign_complete": false,
"hypothesis_results": {
"historical_percentages_reproduced": false,
"qualification": "Current fixed ten-task campaign only; compare arm metrics directly."
}
}
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/experiment_protocol.json
+72
View File
@@ -0,0 +1,72 @@
{
"experiment_id": "2-4",
"protocol_version": "1.0.0",
"frozen_on": "2026-07-30",
"authority": "book/chapter2.md:654",
"benchmark": "vendored tau-bench airline test split",
"provider": "OpenAI",
"model": "gpt-4o-mini",
"user_model": "gpt-4o-mini",
"temperature": 0,
"user_temperature": 0,
"seed": 20260730,
"task_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"trials_per_task": 1,
"max_agent_steps": 30,
"arms": {
"baseline": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_trump": {
"tone": "exaggerated Trump-style",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_casual": {
"tone": "casual with emoji/slang",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"wiki_random": {
"tone": "professional neutral",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "complete"
},
"no_tool_desc": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "all description fields blank"
},
"all_ablations": {
"tone": "casual with emoji/slang",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "all description fields blank"
}
},
"metrics": [
"tau-bench objective reward/pass rate",
"agent steps and model calls",
"tool calls and environment tool errors",
"provider prompt/completion/total tokens",
"provider response IDs",
"LiteLLM native cost estimate"
],
"acceptance_gates": [
"all six arms run the same ten task IDs with the same seed and user simulator",
"each task is scored only by the vendored tau-bench objective environment",
"every agent and simulated-user model call is a real provider response with ID and usage",
"raw request messages, tools, response content/tool calls, usage, timings, and trajectories are retained",
"the randomized wiki contains the same experimental rule material in the frozen pre-generated order",
"the no-description arm preserves schemas while blanking every nested description",
"campaign completion is independent of whether the manuscript hypotheses win",
"historical percentage claims are not treated as reproduced unless this fixed campaign independently yields them",
"credential scan passes"
],
"hypotheses": {
"tone": "tone variants have limited effect on objective task success",
"organization": "randomizing instruction organization reduces success",
"tool_descriptions": "removing descriptions increases tool errors and reduces success"
}
}
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/manifest.json
+33
View File
@@ -0,0 +1,33 @@
{
"experiment_id": "2-4",
"campaign_complete": false,
"protocol_sha256": "ebedaecdbb6c4fc8d8d92222dc17334a27a1b1b9f19e4732578563f5e9389424",
"summary_path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/comparison.json",
"summary_sha256": "0b26332ceba12fc3ab3e5713c4239a382607dd86b76d2e364dc1863067bad19a",
"arm_artifacts": {
"baseline": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-baseline_0730022052.json",
"sha256": "0edf8d281ea5e5d39ca5079d445b1684b1ec8de1026013590d68c5a0e88121fd"
},
"tone_trump": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_trump_0730022059.json",
"sha256": "1531727c765cd3b1caef4b3c486399467979a40f951c603ae577241ed7918ccb"
},
"tone_casual": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_0730022104.json",
"sha256": "4a1205d5ee3394b18e95d184a2f6dc7a565a2c1ba1558428b83f394f3d989b51"
},
"wiki_random": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-wiki_random_0730022109.json",
"sha256": "bca0c69faf8faceabf444cc5f85c51bd7b4db4b72ea83151ed1ea88e60891264"
},
"no_tool_desc": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-no_tool_desc_0730022115.json",
"sha256": "7a0fd841f28b3c6a8fe0f801bb33fba08acbdaa3a3dadbc0330334d209592ebf"
},
"all_ablations": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_wiki_random_no_tool_desc_0730022120.json",
"sha256": "aee798ceabad4e2022adcbadef6aca739937be4fbeffc9f98c9c65950bbd3990"
}
}
}
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-baseline_0730022052.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-no_tool_desc_0730022115.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_0730022104.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_wiki_random_no_tool_desc_0730022120.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_trump_0730022059.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-wiki_random_0730022109.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/comparison.json
+508
View File
@@ -0,0 +1,508 @@
{
"experiment_id": "2-4",
"created_at": "2026-07-30T05:14:32.789700+08:00",
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"protocol_copy": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/experiment_protocol.json",
"provider": "openai",
"model": "kimi-k3",
"user_model": "kimi-k3",
"objective_scoring": "vendored tau-bench environment reward",
"arms": {
"baseline": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-baseline_0730022423.json",
"sha256": "967697b4cb140efa01230c54ce6b177f227c8a617e785071e60c2d4483b39fba"
},
"rewards": [
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 70.0,
"total": 10,
"successes": 7,
"failures": 3,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 25 with role 'user' must not be empty"
}
],
"agent_steps": [
14,
10,
30,
18,
15,
14,
11,
14,
16,
null
],
"tool_calls": [
6,
5,
28,
14,
12,
11,
7,
11,
12,
null
],
"tool_errors": [
1,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 187,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1308761,
"completion_tokens": 71951,
"total_tokens": 1380712
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 33.37032,
"arm_complete": false
},
"tone_trump": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-tone_trump_0730025250.json",
"sha256": "9ed3c56f487b13c9ce47adc46404c4d63d0b28aaf595c4ba42154dcf7e12bfff"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
0.0,
0.0
],
"success_rate": 60.0,
"total": 10,
"successes": 6,
"failures": 4,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 27 with role 'user' must not be empty"
}
],
"agent_steps": [
13,
11,
16,
18,
12,
15,
10,
12,
14,
null
],
"tool_calls": [
7,
7,
14,
14,
9,
12,
6,
8,
10,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 164,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1100695,
"completion_tokens": 78345,
"total_tokens": 1179040
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 29.848399999999998,
"arm_complete": false
},
"tone_casual": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-tone_casual_0730032323.json",
"sha256": "460fe6d9fa11686e4f6b07bf2d1e565c8f12b03cd84f86ec3efc22855ac2aeb5"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 90.0,
"total": 10,
"successes": 9,
"failures": 1,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 7 with role 'user' must not be empty"
}
],
"agent_steps": [
14,
10,
16,
15,
15,
14,
12,
15,
16,
null
],
"tool_calls": [
9,
5,
13,
11,
11,
11,
8,
9,
13,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 173,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1176456,
"completion_tokens": 74474,
"total_tokens": 1250930
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 30.97652,
"arm_complete": false
},
"wiki_random": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-wiki_random_0730035037.json",
"sha256": "645afb3fe2be8cc5cb816b50e4654211586b02eff934d676bf95cb5a0f0d30ff"
},
"rewards": [
1.0,
1.0,
0.0,
0.0,
0.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 60.0,
"total": 10,
"successes": 6,
"failures": 4,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 2,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 7 with role 'user' must not be empty"
},
{
"task_id": 3,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 5 with role 'user' must not be empty"
},
{
"task_id": 9,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 31 with role 'user' must not be empty"
}
],
"agent_steps": [
14,
9,
null,
null,
14,
15,
13,
10,
14,
null
],
"tool_calls": [
7,
5,
null,
null,
10,
13,
10,
7,
10,
null
],
"tool_errors": [
0,
0,
null,
null,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 123,
"response_ids_present": true,
"usage": {
"prompt_tokens": 733921,
"completion_tokens": 54842,
"total_tokens": 788763
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 20.16262,
"arm_complete": false
},
"no_tool_desc": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-no_tool_desc_0730042104.json",
"sha256": "4ee2a432ec120556c4c2b45eafee756ff782b2bd44849f8dba7bfa5da84e70f5"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 80.0,
"total": 10,
"successes": 8,
"failures": 2,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 2,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 5 with role 'user' must not be empty"
},
{
"task_id": 9,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 21 with role 'user' must not be empty"
}
],
"agent_steps": [
14,
9,
null,
15,
12,
15,
11,
13,
13,
null
],
"tool_calls": [
9,
5,
null,
11,
9,
12,
7,
10,
8,
null
],
"tool_errors": [
0,
0,
null,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 141,
"response_ids_present": true,
"usage": {
"prompt_tokens": 788167,
"completion_tokens": 63372,
"total_tokens": 851539
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 22.10054,
"arm_complete": false
},
"all_ablations": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730044439.json",
"sha256": "981a51a7e60a1bd3459b1e0baba7979b5c002b330899741b082b383861a5fa5a"
},
"rewards": [
1.0,
1.0,
1.0,
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 70.0,
"total": 10,
"successes": 7,
"failures": 3,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 3,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 7 with role 'user' must not be empty"
},
{
"task_id": 9,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 29 with role 'user' must not be empty"
}
],
"agent_steps": [
13,
10,
16,
null,
16,
9,
12,
15,
12,
null
],
"tool_calls": [
7,
5,
13,
null,
12,
8,
6,
10,
8,
null
],
"tool_errors": [
0,
0,
0,
null,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 145,
"response_ids_present": true,
"usage": {
"prompt_tokens": 850906,
"completion_tokens": 68218,
"total_tokens": 919124
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 23.83992,
"arm_complete": false
}
},
"credential_scan_passed": true,
"credential_findings": [],
"usage_and_cost": {
"total_real_api_calls": 933,
"prompt_tokens": 5958906,
"completion_tokens": 411202,
"total_tokens": 6370108,
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 160.29832000000002,
"native_cost_complete": true,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
},
"campaign_complete": false,
"hypothesis_results": {
"historical_percentages_reproduced": false,
"qualification": "Current fixed ten-task campaign only; compare arm metrics directly."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/experiment_protocol.json
+94
View File
@@ -0,0 +1,94 @@
{
"experiment_id": "2-4",
"protocol_version": "1.0.0",
"frozen_on": "2026-07-30",
"authority": "book/chapter2.md:654",
"benchmark": "vendored tau-bench airline test split",
"provider": "Moonshot official OpenAI-compatible endpoint",
"model": "kimi-k3",
"user_model": "kimi-k3",
"temperature": 1,
"user_temperature": 1,
"seed": 20260730,
"task_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"trials_per_task": 1,
"max_agent_steps": 30,
"arms": {
"baseline": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_trump": {
"tone": "exaggerated Trump-style",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_casual": {
"tone": "casual with emoji/slang",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"wiki_random": {
"tone": "professional neutral",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "complete"
},
"no_tool_desc": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "all description fields blank"
},
"all_ablations": {
"tone": "casual with emoji/slang",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "all description fields blank"
}
},
"metrics": [
"tau-bench objective reward/pass rate",
"agent steps and model calls",
"tool calls and environment tool errors",
"provider prompt/completion/total tokens",
"provider response IDs",
"LiteLLM native cost estimate"
],
"acceptance_gates": [
"all six arms run the same ten task IDs with the same seed and user simulator",
"each task is scored only by the vendored tau-bench objective environment",
"every agent and simulated-user model call is a real provider response with ID and usage",
"raw request messages, tools, response content/tool calls, usage, timings, and trajectories are retained",
"the randomized wiki contains the same experimental rule material in the frozen pre-generated order",
"the no-description arm preserves schemas while blanking every nested description",
"campaign completion is independent of whether the manuscript hypotheses win",
"historical percentage claims are not treated as reproduced unless this fixed campaign independently yields them",
"credential scan passes"
],
"hypotheses": {
"tone": "tone variants have limited effect on objective task success",
"organization": "randomizing instruction organization reduces success",
"tool_descriptions": "removing descriptions increases tool errors and reduces success"
},
"transport_amendment": {
"amended_on": "2026-07-30",
"reason": "The OpenAI-direct credential returned insufficient_quota and the OpenRouter credential returned User not found before any model response or task observation was obtained.",
"change": "Use the available official Moonshot endpoint and kimi-k3 for both action model and user simulator.",
"unchanged": [
"tasks",
"arms",
"temperature",
"seed",
"step limit",
"objective gates and hypotheses"
],
"failed_campaign": "runs/exp2-4-gpt-4o-mini-20260730-v1",
"methodological_note": "This is an availability amendment, not a response-conditioned change: both rejected transports produced zero successful model calls and therefore exposed no task outcomes."
},
"pricing": {
"currency": "CNY",
"uncached_input_per_million_tokens": 20,
"cached_input_per_million_tokens": 2,
"output_per_million_tokens": 100,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/manifest.json
+33
View File
@@ -0,0 +1,33 @@
{
"experiment_id": "2-4",
"campaign_complete": false,
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"summary_path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/comparison.json",
"summary_sha256": "2f19cc88bbdd0678ac5c31b20fd7f72ed7c9d3b95f9bc42b322da6f39df71242",
"arm_artifacts": {
"baseline": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-baseline_0730022423.json",
"sha256": "967697b4cb140efa01230c54ce6b177f227c8a617e785071e60c2d4483b39fba"
},
"tone_trump": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-tone_trump_0730025250.json",
"sha256": "9ed3c56f487b13c9ce47adc46404c4d63d0b28aaf595c4ba42154dcf7e12bfff"
},
"tone_casual": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-tone_casual_0730032323.json",
"sha256": "460fe6d9fa11686e4f6b07bf2d1e565c8f12b03cd84f86ec3efc22855ac2aeb5"
},
"wiki_random": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-wiki_random_0730035037.json",
"sha256": "645afb3fe2be8cc5cb816b50e4654211586b02eff934d676bf95cb5a0f0d30ff"
},
"no_tool_desc": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-no_tool_desc_0730042104.json",
"sha256": "4ee2a432ec120556c4c2b45eafee756ff782b2bd44849f8dba7bfa5da84e70f5"
},
"all_ablations": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v2/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730044439.json",
"sha256": "981a51a7e60a1bd3459b1e0baba7979b5c002b330899741b082b383861a5fa5a"
}
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/comparison.json
+508
View File
@@ -0,0 +1,508 @@
{
"experiment_id": "2-4",
"created_at": "2026-07-30T05:18:24.493511+08:00",
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"protocol_copy": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/experiment_protocol.json",
"provider": "openai",
"model": "kimi-k3",
"user_model": "kimi-k3",
"objective_scoring": "vendored tau-bench environment reward",
"arms": {
"baseline": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-baseline_0730051815.json",
"sha256": "074fb8b59b8bf1cf1be7de37d2c7b7ccbb9f1db11e58d9a7d0d2cf1889408e86"
},
"rewards": [
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 70.0,
"total": 10,
"successes": 7,
"failures": 3,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.AuthenticationError: AuthenticationError: OpenAIException - Invalid Authentication"
}
],
"agent_steps": [
14,
10,
30,
18,
15,
14,
11,
14,
16,
null
],
"tool_calls": [
6,
5,
28,
14,
12,
11,
7,
11,
12,
null
],
"tool_errors": [
1,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 187,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1308761,
"completion_tokens": 71951,
"total_tokens": 1380712
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 33.37032,
"arm_complete": false
},
"tone_trump": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-tone_trump_0730051817.json",
"sha256": "46471eafdc3ed02bac00c1db82a8d7fce4eee6080c64e5115ec0c798603565c5"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
0.0,
0.0
],
"success_rate": 60.0,
"total": 10,
"successes": 6,
"failures": 4,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.AuthenticationError: AuthenticationError: OpenAIException - Invalid Authentication"
}
],
"agent_steps": [
13,
11,
16,
18,
12,
15,
10,
12,
14,
null
],
"tool_calls": [
7,
7,
14,
14,
9,
12,
6,
8,
10,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 164,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1100695,
"completion_tokens": 78345,
"total_tokens": 1179040
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 29.848399999999998,
"arm_complete": false
},
"tone_casual": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-tone_casual_0730051818.json",
"sha256": "232633b00401c16cf370fc1c68514c2efceaff78d9c6c4d55daed522779d8157"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 90.0,
"total": 10,
"successes": 9,
"failures": 1,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.AuthenticationError: AuthenticationError: OpenAIException - Invalid Authentication"
}
],
"agent_steps": [
14,
10,
16,
15,
15,
14,
12,
15,
16,
null
],
"tool_calls": [
9,
5,
13,
11,
11,
11,
8,
9,
13,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 173,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1176456,
"completion_tokens": 74474,
"total_tokens": 1250930
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 30.97652,
"arm_complete": false
},
"wiki_random": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-wiki_random_0730051819.json",
"sha256": "048d583a054aa87d9664a160163cc5d35c0f6b25ce526cde98e03dfd646c4e61"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
0.0,
0.0,
0.0
],
"success_rate": 60.0,
"total": 10,
"successes": 6,
"failures": 4,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 2,
"error": "litellm.AuthenticationError: AuthenticationError: OpenAIException - Invalid Authentication"
},
{
"task_id": 3,
"error": "litellm.AuthenticationError: AuthenticationError: OpenAIException - Invalid Authentication"
},
{
"task_id": 9,
"error": "litellm.AuthenticationError: AuthenticationError: OpenAIException - Invalid Authentication"
}
],
"agent_steps": [
14,
9,
14,
15,
13,
10,
14,
null,
null,
null
],
"tool_calls": [
7,
5,
10,
13,
10,
7,
10,
null,
null,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
null,
null,
null
],
"real_api_calls": 123,
"response_ids_present": true,
"usage": {
"prompt_tokens": 733921,
"completion_tokens": 54842,
"total_tokens": 788763
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 20.16262,
"arm_complete": false
},
"no_tool_desc": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-no_tool_desc_0730051821.json",
"sha256": "607da9f9ea1e212a044f382b12f04145dcdeab658fe28fb14eb59d0382dbe6da"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0,
0.0
],
"success_rate": 80.0,
"total": 10,
"successes": 8,
"failures": 2,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 2,
"error": "litellm.AuthenticationError: AuthenticationError: OpenAIException - Invalid Authentication"
},
{
"task_id": 9,
"error": "litellm.AuthenticationError: AuthenticationError: OpenAIException - Invalid Authentication"
}
],
"agent_steps": [
14,
9,
15,
12,
15,
11,
13,
13,
null,
null
],
"tool_calls": [
9,
5,
11,
9,
12,
7,
10,
8,
null,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
null,
null
],
"real_api_calls": 141,
"response_ids_present": true,
"usage": {
"prompt_tokens": 788167,
"completion_tokens": 63372,
"total_tokens": 851539
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 22.10054,
"arm_complete": false
},
"all_ablations": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730051822.json",
"sha256": "f1f812e78c4d0eb1c02d2757579a0a8e11e250654b13ebd985e5faf2fdf6f9c6"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
0.0,
0.0
],
"success_rate": 70.0,
"total": 10,
"successes": 7,
"failures": 3,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 3,
"error": "litellm.AuthenticationError: AuthenticationError: OpenAIException - Invalid Authentication"
},
{
"task_id": 9,
"error": "litellm.AuthenticationError: AuthenticationError: OpenAIException - Invalid Authentication"
}
],
"agent_steps": [
13,
10,
16,
16,
9,
12,
15,
12,
null,
null
],
"tool_calls": [
7,
5,
13,
12,
8,
6,
10,
8,
null,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
null,
null
],
"real_api_calls": 145,
"response_ids_present": true,
"usage": {
"prompt_tokens": 850906,
"completion_tokens": 68218,
"total_tokens": 919124
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 23.83992,
"arm_complete": false
}
},
"credential_scan_passed": true,
"credential_findings": [],
"usage_and_cost": {
"total_real_api_calls": 933,
"prompt_tokens": 5958906,
"completion_tokens": 411202,
"total_tokens": 6370108,
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 160.29832000000002,
"native_cost_complete": true,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
},
"campaign_complete": false,
"hypothesis_results": {
"historical_percentages_reproduced": false,
"qualification": "Current fixed ten-task campaign only; compare arm metrics directly."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/experiment_protocol.json
+94
View File
@@ -0,0 +1,94 @@
{
"experiment_id": "2-4",
"protocol_version": "1.0.0",
"frozen_on": "2026-07-30",
"authority": "book/chapter2.md:654",
"benchmark": "vendored tau-bench airline test split",
"provider": "Moonshot official OpenAI-compatible endpoint",
"model": "kimi-k3",
"user_model": "kimi-k3",
"temperature": 1,
"user_temperature": 1,
"seed": 20260730,
"task_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"trials_per_task": 1,
"max_agent_steps": 30,
"arms": {
"baseline": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_trump": {
"tone": "exaggerated Trump-style",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_casual": {
"tone": "casual with emoji/slang",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"wiki_random": {
"tone": "professional neutral",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "complete"
},
"no_tool_desc": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "all description fields blank"
},
"all_ablations": {
"tone": "casual with emoji/slang",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "all description fields blank"
}
},
"metrics": [
"tau-bench objective reward/pass rate",
"agent steps and model calls",
"tool calls and environment tool errors",
"provider prompt/completion/total tokens",
"provider response IDs",
"LiteLLM native cost estimate"
],
"acceptance_gates": [
"all six arms run the same ten task IDs with the same seed and user simulator",
"each task is scored only by the vendored tau-bench objective environment",
"every agent and simulated-user model call is a real provider response with ID and usage",
"raw request messages, tools, response content/tool calls, usage, timings, and trajectories are retained",
"the randomized wiki contains the same experimental rule material in the frozen pre-generated order",
"the no-description arm preserves schemas while blanking every nested description",
"campaign completion is independent of whether the manuscript hypotheses win",
"historical percentage claims are not treated as reproduced unless this fixed campaign independently yields them",
"credential scan passes"
],
"hypotheses": {
"tone": "tone variants have limited effect on objective task success",
"organization": "randomizing instruction organization reduces success",
"tool_descriptions": "removing descriptions increases tool errors and reduces success"
},
"transport_amendment": {
"amended_on": "2026-07-30",
"reason": "The OpenAI-direct credential returned insufficient_quota and the OpenRouter credential returned User not found before any model response or task observation was obtained.",
"change": "Use the available official Moonshot endpoint and kimi-k3 for both action model and user simulator.",
"unchanged": [
"tasks",
"arms",
"temperature",
"seed",
"step limit",
"objective gates and hypotheses"
],
"failed_campaign": "runs/exp2-4-gpt-4o-mini-20260730-v1",
"methodological_note": "This is an availability amendment, not a response-conditioned change: both rejected transports produced zero successful model calls and therefore exposed no task outcomes."
},
"pricing": {
"currency": "CNY",
"uncached_input_per_million_tokens": 20,
"cached_input_per_million_tokens": 2,
"output_per_million_tokens": 100,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/manifest.json
+33
View File
@@ -0,0 +1,33 @@
{
"experiment_id": "2-4",
"campaign_complete": false,
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"summary_path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/comparison.json",
"summary_sha256": "4d17272b5a5eac4a2e4c4945993beb5d9872d3b4fb6a30ee2ecb29fbd0a91996",
"arm_artifacts": {
"baseline": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-baseline_0730051815.json",
"sha256": "074fb8b59b8bf1cf1be7de37d2c7b7ccbb9f1db11e58d9a7d0d2cf1889408e86"
},
"tone_trump": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-tone_trump_0730051817.json",
"sha256": "46471eafdc3ed02bac00c1db82a8d7fce4eee6080c64e5115ec0c798603565c5"
},
"tone_casual": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-tone_casual_0730051818.json",
"sha256": "232633b00401c16cf370fc1c68514c2efceaff78d9c6c4d55daed522779d8157"
},
"wiki_random": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-wiki_random_0730051819.json",
"sha256": "048d583a054aa87d9664a160163cc5d35c0f6b25ce526cde98e03dfd646c4e61"
},
"no_tool_desc": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-no_tool_desc_0730051821.json",
"sha256": "607da9f9ea1e212a044f382b12f04145dcdeab658fe28fb14eb59d0382dbe6da"
},
"all_ablations": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v3/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730051822.json",
"sha256": "f1f812e78c4d0eb1c02d2757579a0a8e11e250654b13ebd985e5faf2fdf6f9c6"
}
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/comparison.json
+477
View File
@@ -0,0 +1,477 @@
{
"experiment_id": "2-4",
"created_at": "2026-07-30T06:13:09.682117+08:00",
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"protocol_copy": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/experiment_protocol.json",
"provider": "openai",
"model": "kimi-k3",
"user_model": "kimi-k3",
"objective_scoring": "vendored tau-bench environment reward",
"arms": {
"baseline": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-baseline_0730051848.json",
"sha256": "0adf5fe03e21908792e4ae1218c1ba0500c6b7942c8ee3cc679f36426513727e"
},
"rewards": [
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 70.0,
"total": 10,
"successes": 7,
"failures": 3,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
10,
30,
18,
15,
14,
11,
14,
16,
24
],
"tool_calls": [
6,
5,
28,
14,
12,
11,
7,
11,
12,
17
],
"tool_errors": [
1,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 221,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1599454,
"completion_tokens": 88017,
"total_tokens": 1687471
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 40.79078,
"arm_complete": true
},
"tone_trump": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-tone_trump_0730052734.json",
"sha256": "3c9766ed2801aa70003b24cc013dcbf6997581236f76700798a591c28b1a0293"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
0.0,
0.0
],
"success_rate": 60.0,
"total": 10,
"successes": 6,
"failures": 4,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "User simulator returned empty content on three accepted responses"
}
],
"agent_steps": [
13,
11,
16,
18,
12,
15,
10,
12,
14,
null
],
"tool_calls": [
7,
7,
14,
14,
9,
12,
6,
8,
10,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 171,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1108201,
"completion_tokens": 81717,
"total_tokens": 1189918
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 30.335720000000002,
"arm_complete": false
},
"tone_casual": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-tone_casual_0730053331.json",
"sha256": "8a07bcd0041469d53bc7228deaf02eb54edf350dbe7460c37cf5fca6e3a6e654"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 90.0,
"total": 10,
"successes": 9,
"failures": 1,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 13 with role 'user' must not be empty"
}
],
"agent_steps": [
14,
10,
16,
15,
15,
14,
12,
15,
16,
null
],
"tool_calls": [
9,
5,
13,
11,
11,
11,
8,
9,
13,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 180,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1187766,
"completion_tokens": 77626,
"total_tokens": 1265392
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 31.51792,
"arm_complete": false
},
"wiki_random": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-wiki_random_0730054009.json",
"sha256": "dc3c8d20d531e7d555d56325ac2b40989316f65f1cb875a499a6a5331c62a522"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 80.0,
"total": 10,
"successes": 8,
"failures": 2,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
9,
14,
15,
13,
10,
14,
30,
15,
23
],
"tool_calls": [
7,
5,
10,
13,
10,
7,
10,
28,
11,
15
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 209,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1446738,
"completion_tokens": 89732,
"total_tokens": 1536470
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 37.90796,
"arm_complete": true
},
"no_tool_desc": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-no_tool_desc_0730055428.json",
"sha256": "590af99d99429ab69df3a46b9e895ac654f1697534a685e9891794c9d8471c9a"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 90.0,
"total": 10,
"successes": 9,
"failures": 1,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "User simulator returned empty content on three accepted responses"
}
],
"agent_steps": [
14,
9,
15,
12,
15,
11,
13,
13,
17,
null
],
"tool_calls": [
9,
5,
11,
9,
12,
7,
10,
8,
14,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 172,
"response_ids_present": true,
"usage": {
"prompt_tokens": 967540,
"completion_tokens": 80968,
"total_tokens": 1048508
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 27.4476,
"arm_complete": false
},
"all_ablations": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730060254.json",
"sha256": "9bc93f263316cba3ee94ebd55311aa297d67dfd2f359c926e012cde5b9d2000b"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 80.0,
"total": 10,
"successes": 8,
"failures": 2,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
13,
10,
16,
16,
9,
12,
15,
12,
15,
21
],
"tool_calls": [
7,
5,
13,
12,
8,
6,
10,
8,
10,
14
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 196,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1247279,
"completion_tokens": 97489,
"total_tokens": 1344768
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 34.69448,
"arm_complete": true
}
},
"credential_scan_passed": true,
"credential_findings": [],
"usage_and_cost": {
"total_real_api_calls": 1149,
"prompt_tokens": 7556978,
"completion_tokens": 515549,
"total_tokens": 8072527,
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 202.69446,
"native_cost_complete": true,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
},
"campaign_complete": false,
"hypothesis_results": {
"historical_percentages_reproduced": false,
"qualification": "Current fixed ten-task campaign only; compare arm metrics directly."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/experiment_protocol.json
+94
View File
@@ -0,0 +1,94 @@
{
"experiment_id": "2-4",
"protocol_version": "1.0.0",
"frozen_on": "2026-07-30",
"authority": "book/chapter2.md:654",
"benchmark": "vendored tau-bench airline test split",
"provider": "Moonshot official OpenAI-compatible endpoint",
"model": "kimi-k3",
"user_model": "kimi-k3",
"temperature": 1,
"user_temperature": 1,
"seed": 20260730,
"task_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"trials_per_task": 1,
"max_agent_steps": 30,
"arms": {
"baseline": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_trump": {
"tone": "exaggerated Trump-style",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_casual": {
"tone": "casual with emoji/slang",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"wiki_random": {
"tone": "professional neutral",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "complete"
},
"no_tool_desc": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "all description fields blank"
},
"all_ablations": {
"tone": "casual with emoji/slang",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "all description fields blank"
}
},
"metrics": [
"tau-bench objective reward/pass rate",
"agent steps and model calls",
"tool calls and environment tool errors",
"provider prompt/completion/total tokens",
"provider response IDs",
"LiteLLM native cost estimate"
],
"acceptance_gates": [
"all six arms run the same ten task IDs with the same seed and user simulator",
"each task is scored only by the vendored tau-bench objective environment",
"every agent and simulated-user model call is a real provider response with ID and usage",
"raw request messages, tools, response content/tool calls, usage, timings, and trajectories are retained",
"the randomized wiki contains the same experimental rule material in the frozen pre-generated order",
"the no-description arm preserves schemas while blanking every nested description",
"campaign completion is independent of whether the manuscript hypotheses win",
"historical percentage claims are not treated as reproduced unless this fixed campaign independently yields them",
"credential scan passes"
],
"hypotheses": {
"tone": "tone variants have limited effect on objective task success",
"organization": "randomizing instruction organization reduces success",
"tool_descriptions": "removing descriptions increases tool errors and reduces success"
},
"transport_amendment": {
"amended_on": "2026-07-30",
"reason": "The OpenAI-direct credential returned insufficient_quota and the OpenRouter credential returned User not found before any model response or task observation was obtained.",
"change": "Use the available official Moonshot endpoint and kimi-k3 for both action model and user simulator.",
"unchanged": [
"tasks",
"arms",
"temperature",
"seed",
"step limit",
"objective gates and hypotheses"
],
"failed_campaign": "runs/exp2-4-gpt-4o-mini-20260730-v1",
"methodological_note": "This is an availability amendment, not a response-conditioned change: both rejected transports produced zero successful model calls and therefore exposed no task outcomes."
},
"pricing": {
"currency": "CNY",
"uncached_input_per_million_tokens": 20,
"cached_input_per_million_tokens": 2,
"output_per_million_tokens": 100,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/manifest.json
+33
View File
@@ -0,0 +1,33 @@
{
"experiment_id": "2-4",
"campaign_complete": false,
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"summary_path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/comparison.json",
"summary_sha256": "075224a712900ff04cf37df815961e5615a7347c834aa43669cf33411bde83f2",
"arm_artifacts": {
"baseline": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-baseline_0730051848.json",
"sha256": "0adf5fe03e21908792e4ae1218c1ba0500c6b7942c8ee3cc679f36426513727e"
},
"tone_trump": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-tone_trump_0730052734.json",
"sha256": "3c9766ed2801aa70003b24cc013dcbf6997581236f76700798a591c28b1a0293"
},
"tone_casual": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-tone_casual_0730053331.json",
"sha256": "8a07bcd0041469d53bc7228deaf02eb54edf350dbe7460c37cf5fca6e3a6e654"
},
"wiki_random": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-wiki_random_0730054009.json",
"sha256": "dc3c8d20d531e7d555d56325ac2b40989316f65f1cb875a499a6a5331c62a522"
},
"no_tool_desc": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-no_tool_desc_0730055428.json",
"sha256": "590af99d99429ab69df3a46b9e895ac654f1697534a685e9891794c9d8471c9a"
},
"all_ablations": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v4/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730060254.json",
"sha256": "9bc93f263316cba3ee94ebd55311aa297d67dfd2f359c926e012cde5b9d2000b"
}
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/comparison.json
+477
View File
@@ -0,0 +1,477 @@
{
"experiment_id": "2-4",
"created_at": "2026-07-30T06:23:29.421682+08:00",
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"protocol_copy": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/experiment_protocol.json",
"provider": "openai",
"model": "kimi-k3",
"user_model": "kimi-k3",
"objective_scoring": "vendored tau-bench environment reward",
"arms": {
"baseline": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-baseline_0730062324.json",
"sha256": "c2bcc6260fe07b8c427131aed29a16c7c23f0351d0a1db4ce85bdd66b9a6bf38"
},
"rewards": [
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 70.0,
"total": 10,
"successes": 7,
"failures": 3,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
10,
30,
18,
15,
14,
11,
14,
16,
24
],
"tool_calls": [
6,
5,
28,
14,
12,
11,
7,
11,
12,
17
],
"tool_errors": [
1,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 221,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1599454,
"completion_tokens": 88017,
"total_tokens": 1687471
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 40.79078,
"arm_complete": true
},
"tone_trump": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-tone_trump_0730062324.json",
"sha256": "e5b2328f6e49c8959cd45bbd9fb30fbdd9e2ab1df7bfffb6c7e20ed83c1b1ac2"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
0.0,
0.0
],
"success_rate": 60.0,
"total": 10,
"successes": 6,
"failures": 4,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.NotFoundError: OpenAIException - The model `kimi-k3` does not exist or you do not have access to it."
}
],
"agent_steps": [
13,
11,
16,
18,
12,
15,
10,
12,
14,
null
],
"tool_calls": [
7,
7,
14,
14,
9,
12,
6,
8,
10,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 164,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1100695,
"completion_tokens": 78345,
"total_tokens": 1179040
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 29.848399999999998,
"arm_complete": false
},
"tone_casual": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-tone_casual_0730062326.json",
"sha256": "3bb24ed4bc8ce8fec1808dad703c922235cbfa2972ad4b54c5c76da378204038"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 90.0,
"total": 10,
"successes": 9,
"failures": 1,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.NotFoundError: OpenAIException - The model `kimi-k3` does not exist or you do not have access to it."
}
],
"agent_steps": [
14,
10,
16,
15,
15,
14,
12,
15,
16,
null
],
"tool_calls": [
9,
5,
13,
11,
11,
11,
8,
9,
13,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 173,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1176456,
"completion_tokens": 74474,
"total_tokens": 1250930
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 30.97652,
"arm_complete": false
},
"wiki_random": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-wiki_random_0730062327.json",
"sha256": "db1d386704fca08b55d8a3f4cca46976e23ec97950702c63de5d443bd2354820"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 80.0,
"total": 10,
"successes": 8,
"failures": 2,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
9,
14,
15,
13,
10,
14,
30,
15,
23
],
"tool_calls": [
7,
5,
10,
13,
10,
7,
10,
28,
11,
15
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 209,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1446738,
"completion_tokens": 89732,
"total_tokens": 1536470
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 37.90796,
"arm_complete": true
},
"no_tool_desc": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-no_tool_desc_0730062327.json",
"sha256": "fa003ae5dccfbc58d338d7973434de7a2ca4e3bc436dac6970031c5d925d20ed"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 90.0,
"total": 10,
"successes": 9,
"failures": 1,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.NotFoundError: OpenAIException - The model `kimi-k3` does not exist or you do not have access to it."
}
],
"agent_steps": [
14,
9,
15,
12,
15,
11,
13,
13,
17,
null
],
"tool_calls": [
9,
5,
11,
9,
12,
7,
10,
8,
14,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 162,
"response_ids_present": true,
"usage": {
"prompt_tokens": 944131,
"completion_tokens": 74801,
"total_tokens": 1018932
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 26.36272,
"arm_complete": false
},
"all_ablations": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730062328.json",
"sha256": "98733b43f7c1d41932bcceebd47dca3c0b9257da0cb6affdc3205273eb464d13"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 80.0,
"total": 10,
"successes": 8,
"failures": 2,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
13,
10,
16,
16,
9,
12,
15,
12,
15,
21
],
"tool_calls": [
7,
5,
13,
12,
8,
6,
10,
8,
10,
14
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 196,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1247279,
"completion_tokens": 97489,
"total_tokens": 1344768
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 34.69448,
"arm_complete": true
}
},
"credential_scan_passed": true,
"credential_findings": [],
"usage_and_cost": {
"total_real_api_calls": 1125,
"prompt_tokens": 7514753,
"completion_tokens": 502858,
"total_tokens": 8017611,
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 200.58086,
"native_cost_complete": true,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
},
"campaign_complete": false,
"hypothesis_results": {
"historical_percentages_reproduced": false,
"qualification": "Current fixed ten-task campaign only; compare arm metrics directly."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/experiment_protocol.json
+94
View File
@@ -0,0 +1,94 @@
{
"experiment_id": "2-4",
"protocol_version": "1.0.0",
"frozen_on": "2026-07-30",
"authority": "book/chapter2.md:654",
"benchmark": "vendored tau-bench airline test split",
"provider": "Moonshot official OpenAI-compatible endpoint",
"model": "kimi-k3",
"user_model": "kimi-k3",
"temperature": 1,
"user_temperature": 1,
"seed": 20260730,
"task_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"trials_per_task": 1,
"max_agent_steps": 30,
"arms": {
"baseline": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_trump": {
"tone": "exaggerated Trump-style",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_casual": {
"tone": "casual with emoji/slang",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"wiki_random": {
"tone": "professional neutral",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "complete"
},
"no_tool_desc": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "all description fields blank"
},
"all_ablations": {
"tone": "casual with emoji/slang",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "all description fields blank"
}
},
"metrics": [
"tau-bench objective reward/pass rate",
"agent steps and model calls",
"tool calls and environment tool errors",
"provider prompt/completion/total tokens",
"provider response IDs",
"LiteLLM native cost estimate"
],
"acceptance_gates": [
"all six arms run the same ten task IDs with the same seed and user simulator",
"each task is scored only by the vendored tau-bench objective environment",
"every agent and simulated-user model call is a real provider response with ID and usage",
"raw request messages, tools, response content/tool calls, usage, timings, and trajectories are retained",
"the randomized wiki contains the same experimental rule material in the frozen pre-generated order",
"the no-description arm preserves schemas while blanking every nested description",
"campaign completion is independent of whether the manuscript hypotheses win",
"historical percentage claims are not treated as reproduced unless this fixed campaign independently yields them",
"credential scan passes"
],
"hypotheses": {
"tone": "tone variants have limited effect on objective task success",
"organization": "randomizing instruction organization reduces success",
"tool_descriptions": "removing descriptions increases tool errors and reduces success"
},
"transport_amendment": {
"amended_on": "2026-07-30",
"reason": "The OpenAI-direct credential returned insufficient_quota and the OpenRouter credential returned User not found before any model response or task observation was obtained.",
"change": "Use the available official Moonshot endpoint and kimi-k3 for both action model and user simulator.",
"unchanged": [
"tasks",
"arms",
"temperature",
"seed",
"step limit",
"objective gates and hypotheses"
],
"failed_campaign": "runs/exp2-4-gpt-4o-mini-20260730-v1",
"methodological_note": "This is an availability amendment, not a response-conditioned change: both rejected transports produced zero successful model calls and therefore exposed no task outcomes."
},
"pricing": {
"currency": "CNY",
"uncached_input_per_million_tokens": 20,
"cached_input_per_million_tokens": 2,
"output_per_million_tokens": 100,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/manifest.json
+33
View File
@@ -0,0 +1,33 @@
{
"experiment_id": "2-4",
"campaign_complete": false,
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"summary_path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/comparison.json",
"summary_sha256": "eb1dc1dc1bec466f6815fc08093d555e18e31491c3ce8a74fbaec7745c7987ff",
"arm_artifacts": {
"baseline": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-baseline_0730062324.json",
"sha256": "c2bcc6260fe07b8c427131aed29a16c7c23f0351d0a1db4ce85bdd66b9a6bf38"
},
"tone_trump": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-tone_trump_0730062324.json",
"sha256": "e5b2328f6e49c8959cd45bbd9fb30fbdd9e2ab1df7bfffb6c7e20ed83c1b1ac2"
},
"tone_casual": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-tone_casual_0730062326.json",
"sha256": "3bb24ed4bc8ce8fec1808dad703c922235cbfa2972ad4b54c5c76da378204038"
},
"wiki_random": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-wiki_random_0730062327.json",
"sha256": "db1d386704fca08b55d8a3f4cca46976e23ec97950702c63de5d443bd2354820"
},
"no_tool_desc": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-no_tool_desc_0730062327.json",
"sha256": "fa003ae5dccfbc58d338d7973434de7a2ca4e3bc436dac6970031c5d925d20ed"
},
"all_ablations": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v5/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730062328.json",
"sha256": "98733b43f7c1d41932bcceebd47dca3c0b9257da0cb6affdc3205273eb464d13"
}
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/comparison.json
+467
View File
@@ -0,0 +1,467 @@
{
"experiment_id": "2-4",
"created_at": "2026-07-30T06:52:16.667084+08:00",
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"protocol_copy": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/experiment_protocol.json",
"provider": "openai",
"model": "kimi-k3",
"user_model": "kimi-k3",
"objective_scoring": "vendored tau-bench environment reward",
"arms": {
"baseline": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-baseline_0730062910.json",
"sha256": "33dfed80028e3c0fe8ec1984e4441cbfe54b90ac7cd5f4a88dd3caaedb1016b8"
},
"rewards": [
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 70.0,
"total": 10,
"successes": 7,
"failures": 3,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
10,
30,
18,
15,
14,
11,
14,
16,
24
],
"tool_calls": [
6,
5,
28,
14,
12,
11,
7,
11,
12,
17
],
"tool_errors": [
1,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 221,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1599454,
"completion_tokens": 88017,
"total_tokens": 1687471
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 40.79078,
"arm_complete": true
},
"tone_trump": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-tone_trump_0730062910.json",
"sha256": "19f63afb3c31963a857181bc177917b3382eb900296518e69a5d0ac36078e0b9"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
0.0,
0.0
],
"success_rate": 60.0,
"total": 10,
"successes": 6,
"failures": 4,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
13,
11,
16,
18,
12,
15,
10,
12,
14,
23
],
"tool_calls": [
7,
7,
14,
14,
9,
12,
6,
8,
10,
17
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 194,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1425756,
"completion_tokens": 94297,
"total_tokens": 1520053
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 37.94482,
"arm_complete": true
},
"tone_casual": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-tone_casual_0730063807.json",
"sha256": "f5c8b66b3de895d2d9dc5d514ccc1ba28400ac6ef7b97ded605e23d247d9b84e"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 90.0,
"total": 10,
"successes": 9,
"failures": 1,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
10,
16,
15,
15,
14,
12,
15,
16,
19
],
"tool_calls": [
9,
5,
13,
11,
11,
11,
8,
9,
13,
12
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 200,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1435720,
"completion_tokens": 93507,
"total_tokens": 1529227
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 38.0651,
"arm_complete": true
},
"wiki_random": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-wiki_random_0730064805.json",
"sha256": "ce1f89c6f23f627a075edaf56a3302698507e4051dfd8e38aaa82e8c3af30a8a"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 80.0,
"total": 10,
"successes": 8,
"failures": 2,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
9,
14,
15,
13,
10,
14,
30,
15,
23
],
"tool_calls": [
7,
5,
10,
13,
10,
7,
10,
28,
11,
15
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 209,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1446738,
"completion_tokens": 89732,
"total_tokens": 1536470
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 37.90796,
"arm_complete": true
},
"no_tool_desc": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-no_tool_desc_0730064805.json",
"sha256": "2fa1536d6df839390ed201995d9ae517c168e1659ebaaa7ee26c9a26028c30a6"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 90.0,
"total": 10,
"successes": 9,
"failures": 1,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 9,
"error": "litellm.BadRequestError: OpenAIException - Invalid request: the message at position 9 with role 'user' must not be empty"
}
],
"agent_steps": [
14,
9,
15,
12,
15,
11,
13,
13,
17,
null
],
"tool_calls": [
9,
5,
11,
9,
12,
7,
10,
8,
14,
null
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
null
],
"real_api_calls": 166,
"response_ids_present": true,
"usage": {
"prompt_tokens": 947675,
"completion_tokens": 75364,
"total_tokens": 1023039
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 26.4899,
"arm_complete": false
},
"all_ablations": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730065215.json",
"sha256": "734fedd1db14660de097ac20b028808e12f4033c1fec1db64ac66948789f55a1"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 80.0,
"total": 10,
"successes": 8,
"failures": 2,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
13,
10,
16,
16,
9,
12,
15,
12,
15,
21
],
"tool_calls": [
7,
5,
13,
12,
8,
6,
10,
8,
10,
14
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 196,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1247279,
"completion_tokens": 97489,
"total_tokens": 1344768
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 34.69448,
"arm_complete": true
}
},
"credential_scan_passed": true,
"credential_findings": [],
"usage_and_cost": {
"total_real_api_calls": 1186,
"prompt_tokens": 8102622,
"completion_tokens": 538406,
"total_tokens": 8641028,
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 215.89304,
"native_cost_complete": true,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
},
"campaign_complete": false,
"hypothesis_results": {
"historical_percentages_reproduced": false,
"qualification": "Current fixed ten-task campaign only; compare arm metrics directly."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/experiment_protocol.json
+94
View File
@@ -0,0 +1,94 @@
{
"experiment_id": "2-4",
"protocol_version": "1.0.0",
"frozen_on": "2026-07-30",
"authority": "book/chapter2.md:654",
"benchmark": "vendored tau-bench airline test split",
"provider": "Moonshot official OpenAI-compatible endpoint",
"model": "kimi-k3",
"user_model": "kimi-k3",
"temperature": 1,
"user_temperature": 1,
"seed": 20260730,
"task_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"trials_per_task": 1,
"max_agent_steps": 30,
"arms": {
"baseline": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_trump": {
"tone": "exaggerated Trump-style",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_casual": {
"tone": "casual with emoji/slang",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"wiki_random": {
"tone": "professional neutral",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "complete"
},
"no_tool_desc": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "all description fields blank"
},
"all_ablations": {
"tone": "casual with emoji/slang",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "all description fields blank"
}
},
"metrics": [
"tau-bench objective reward/pass rate",
"agent steps and model calls",
"tool calls and environment tool errors",
"provider prompt/completion/total tokens",
"provider response IDs",
"LiteLLM native cost estimate"
],
"acceptance_gates": [
"all six arms run the same ten task IDs with the same seed and user simulator",
"each task is scored only by the vendored tau-bench objective environment",
"every agent and simulated-user model call is a real provider response with ID and usage",
"raw request messages, tools, response content/tool calls, usage, timings, and trajectories are retained",
"the randomized wiki contains the same experimental rule material in the frozen pre-generated order",
"the no-description arm preserves schemas while blanking every nested description",
"campaign completion is independent of whether the manuscript hypotheses win",
"historical percentage claims are not treated as reproduced unless this fixed campaign independently yields them",
"credential scan passes"
],
"hypotheses": {
"tone": "tone variants have limited effect on objective task success",
"organization": "randomizing instruction organization reduces success",
"tool_descriptions": "removing descriptions increases tool errors and reduces success"
},
"transport_amendment": {
"amended_on": "2026-07-30",
"reason": "The OpenAI-direct credential returned insufficient_quota and the OpenRouter credential returned User not found before any model response or task observation was obtained.",
"change": "Use the available official Moonshot endpoint and kimi-k3 for both action model and user simulator.",
"unchanged": [
"tasks",
"arms",
"temperature",
"seed",
"step limit",
"objective gates and hypotheses"
],
"failed_campaign": "runs/exp2-4-gpt-4o-mini-20260730-v1",
"methodological_note": "This is an availability amendment, not a response-conditioned change: both rejected transports produced zero successful model calls and therefore exposed no task outcomes."
},
"pricing": {
"currency": "CNY",
"uncached_input_per_million_tokens": 20,
"cached_input_per_million_tokens": 2,
"output_per_million_tokens": 100,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/manifest.json
+33
View File
@@ -0,0 +1,33 @@
{
"experiment_id": "2-4",
"campaign_complete": false,
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"summary_path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/comparison.json",
"summary_sha256": "ca3911032da3785075ba9b5921360092e68908d40facd8f8b216c3fd5fdb3886",
"arm_artifacts": {
"baseline": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-baseline_0730062910.json",
"sha256": "33dfed80028e3c0fe8ec1984e4441cbfe54b90ac7cd5f4a88dd3caaedb1016b8"
},
"tone_trump": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-tone_trump_0730062910.json",
"sha256": "19f63afb3c31963a857181bc177917b3382eb900296518e69a5d0ac36078e0b9"
},
"tone_casual": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-tone_casual_0730063807.json",
"sha256": "f5c8b66b3de895d2d9dc5d514ccc1ba28400ac6ef7b97ded605e23d247d9b84e"
},
"wiki_random": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-wiki_random_0730064805.json",
"sha256": "ce1f89c6f23f627a075edaf56a3302698507e4051dfd8e38aaa82e8c3af30a8a"
},
"no_tool_desc": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-no_tool_desc_0730064805.json",
"sha256": "2fa1536d6df839390ed201995d9ae517c168e1659ebaaa7ee26c9a26028c30a6"
},
"all_ablations": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v6/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730065215.json",
"sha256": "734fedd1db14660de097ac20b028808e12f4033c1fec1db64ac66948789f55a1"
}
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/comparison.json
+462
View File
@@ -0,0 +1,462 @@
{
"experiment_id": "2-4",
"created_at": "2026-07-30T12:26:22.987790+08:00",
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"protocol_copy": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/experiment_protocol.json",
"provider": "openai",
"model": "kimi-k3",
"user_model": "kimi-k3",
"objective_scoring": "vendored tau-bench environment reward",
"arms": {
"baseline": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-baseline_0730121451.json",
"sha256": "ff5388e01597814644bb329caab1837fd44779555ceed84ed28c161b575ad056"
},
"rewards": [
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 70.0,
"total": 10,
"successes": 7,
"failures": 3,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
10,
30,
18,
15,
14,
11,
14,
16,
24
],
"tool_calls": [
6,
5,
28,
14,
12,
11,
7,
11,
12,
17
],
"tool_errors": [
1,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 221,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1599454,
"completion_tokens": 88017,
"total_tokens": 1687471
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 40.79078,
"arm_complete": true
},
"tone_trump": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-tone_trump_0730121452.json",
"sha256": "4450b537f8b2d41178f71c6aaaec6ae5ac3feac7bd469f101d0e7699ec3eee51"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
0.0,
1.0,
1.0,
1.0,
0.0,
0.0
],
"success_rate": 60.0,
"total": 10,
"successes": 6,
"failures": 4,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
13,
11,
16,
18,
12,
15,
10,
12,
14,
23
],
"tool_calls": [
7,
7,
14,
14,
9,
12,
6,
8,
10,
17
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 194,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1425756,
"completion_tokens": 94297,
"total_tokens": 1520053
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 37.94482,
"arm_complete": true
},
"tone_casual": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-tone_casual_0730121452.json",
"sha256": "792a087353695f4ac3e4007d9fa6ffebc7b54c85c1249ca728524e95b292e502"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 90.0,
"total": 10,
"successes": 9,
"failures": 1,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
10,
16,
15,
15,
14,
12,
15,
16,
19
],
"tool_calls": [
9,
5,
13,
11,
11,
11,
8,
9,
13,
12
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 200,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1435720,
"completion_tokens": 93507,
"total_tokens": 1529227
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 38.0651,
"arm_complete": true
},
"wiki_random": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-wiki_random_0730121452.json",
"sha256": "b071977829c53deeb1b03ae01017ad2341f247c1028781dfd1d5a4d3de2936a6"
},
"rewards": [
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 80.0,
"total": 10,
"successes": 8,
"failures": 2,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
9,
14,
15,
13,
10,
14,
30,
15,
23
],
"tool_calls": [
7,
5,
10,
13,
10,
7,
10,
28,
11,
15
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 209,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1446738,
"completion_tokens": 89732,
"total_tokens": 1536470
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 37.90796,
"arm_complete": true
},
"no_tool_desc": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-no_tool_desc_0730121452.json",
"sha256": "dfc27bdabf2d15c82f244d82f4a25b40349692b8d6fa66fd6acd9560a9731e4b"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 90.0,
"total": 10,
"successes": 9,
"failures": 1,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
14,
9,
15,
12,
15,
11,
13,
13,
17,
18
],
"tool_calls": [
9,
5,
11,
9,
12,
7,
10,
8,
14,
12
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 187,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1171788,
"completion_tokens": 93672,
"total_tokens": 1265460
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 32.80296,
"arm_complete": true
},
"all_ablations": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730122622.json",
"sha256": "94e7428c6589171096f24cb0d16de10df2c4af0b5e73a559e9df8003d6100dce"
},
"rewards": [
1.0,
1.0,
1.0,
1.0,
0.0,
1.0,
1.0,
1.0,
1.0,
0.0
],
"success_rate": 80.0,
"total": 10,
"successes": 8,
"failures": 2,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [],
"agent_steps": [
13,
10,
16,
16,
9,
12,
15,
12,
15,
21
],
"tool_calls": [
7,
5,
13,
12,
8,
6,
10,
8,
10,
14
],
"tool_errors": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"real_api_calls": 196,
"response_ids_present": true,
"usage": {
"prompt_tokens": 1247279,
"completion_tokens": 97489,
"total_tokens": 1344768
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 34.69448,
"arm_complete": true
}
},
"credential_scan_passed": true,
"credential_findings": [],
"usage_and_cost": {
"total_real_api_calls": 1207,
"prompt_tokens": 8326735,
"completion_tokens": 556714,
"total_tokens": 8883449,
"observed_litellm_cost_usd": 0,
"all_calls_priced": false,
"native_cost_cny": 222.2061,
"native_cost_complete": true,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
},
"campaign_complete": true,
"hypothesis_results": {
"historical_percentages_reproduced": false,
"qualification": "Current fixed ten-task campaign only; compare arm metrics directly."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/experiment_protocol.json
+94
View File
@@ -0,0 +1,94 @@
{
"experiment_id": "2-4",
"protocol_version": "1.0.0",
"frozen_on": "2026-07-30",
"authority": "book/chapter2.md:654",
"benchmark": "vendored tau-bench airline test split",
"provider": "Moonshot official OpenAI-compatible endpoint",
"model": "kimi-k3",
"user_model": "kimi-k3",
"temperature": 1,
"user_temperature": 1,
"seed": 20260730,
"task_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"trials_per_task": 1,
"max_agent_steps": 30,
"arms": {
"baseline": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_trump": {
"tone": "exaggerated Trump-style",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_casual": {
"tone": "casual with emoji/slang",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"wiki_random": {
"tone": "professional neutral",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "complete"
},
"no_tool_desc": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "all description fields blank"
},
"all_ablations": {
"tone": "casual with emoji/slang",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "all description fields blank"
}
},
"metrics": [
"tau-bench objective reward/pass rate",
"agent steps and model calls",
"tool calls and environment tool errors",
"provider prompt/completion/total tokens",
"provider response IDs",
"LiteLLM native cost estimate"
],
"acceptance_gates": [
"all six arms run the same ten task IDs with the same seed and user simulator",
"each task is scored only by the vendored tau-bench objective environment",
"every agent and simulated-user model call is a real provider response with ID and usage",
"raw request messages, tools, response content/tool calls, usage, timings, and trajectories are retained",
"the randomized wiki contains the same experimental rule material in the frozen pre-generated order",
"the no-description arm preserves schemas while blanking every nested description",
"campaign completion is independent of whether the manuscript hypotheses win",
"historical percentage claims are not treated as reproduced unless this fixed campaign independently yields them",
"credential scan passes"
],
"hypotheses": {
"tone": "tone variants have limited effect on objective task success",
"organization": "randomizing instruction organization reduces success",
"tool_descriptions": "removing descriptions increases tool errors and reduces success"
},
"transport_amendment": {
"amended_on": "2026-07-30",
"reason": "The OpenAI-direct credential returned insufficient_quota and the OpenRouter credential returned User not found before any model response or task observation was obtained.",
"change": "Use the available official Moonshot endpoint and kimi-k3 for both action model and user simulator.",
"unchanged": [
"tasks",
"arms",
"temperature",
"seed",
"step limit",
"objective gates and hypotheses"
],
"failed_campaign": "runs/exp2-4-gpt-4o-mini-20260730-v1",
"methodological_note": "This is an availability amendment, not a response-conditioned change: both rejected transports produced zero successful model calls and therefore exposed no task outcomes."
},
"pricing": {
"currency": "CNY",
"uncached_input_per_million_tokens": 20,
"cached_input_per_million_tokens": 2,
"output_per_million_tokens": 100,
"qualification": "Prompt cache detail is unavailable in these responses, so every prompt token is conservatively priced as uncached."
}
}
chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/manifest.json
+33
View File
@@ -0,0 +1,33 @@
{
"experiment_id": "2-4",
"campaign_complete": true,
"protocol_sha256": "ec1ec36d5a6ee21b5185295a7a4447f4e0173f1681a9fe471b240dda1e7cf32a",
"summary_path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/comparison.json",
"summary_sha256": "64d4218ce721bd51c58fce4314ffc62fa8e8fd66975b5b4be54051a64be8a63a",
"arm_artifacts": {
"baseline": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-baseline_0730121451.json",
"sha256": "ff5388e01597814644bb329caab1837fd44779555ceed84ed28c161b575ad056"
},
"tone_trump": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-tone_trump_0730121452.json",
"sha256": "4450b537f8b2d41178f71c6aaaec6ae5ac3feac7bd469f101d0e7699ec3eee51"
},
"tone_casual": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-tone_casual_0730121452.json",
"sha256": "792a087353695f4ac3e4007d9fa6ffebc7b54c85c1249ca728524e95b292e502"
},
"wiki_random": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-wiki_random_0730121452.json",
"sha256": "b071977829c53deeb1b03ae01017ad2341f247c1028781dfd1d5a4d3de2936a6"
},
"no_tool_desc": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-no_tool_desc_0730121452.json",
"sha256": "dfc27bdabf2d15c82f244d82f4a25b40349692b8d6fa66fd6acd9560a9731e4b"
},
"all_ablations": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-kimi-k3-20260730-v7/tool-calling-kimi-k3-tone_casual_wiki_random_no_tool_desc_0730122622.json",
"sha256": "94e7428c6589171096f24cb0d16de10df2c4af0b5e73a559e9df8003d6100dce"
}
}
}