liqiang/ai-agent-book
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
Details
dependency resolution / resolve (3.11) (push) Canceled after 0s
Details
dependency resolution / resolve (3.13) (push) Canceled after 0s
Details
deploy-pages / build (push) Canceled after 0s
Details
deploy-pages / deploy (push) Canceled after 0s
Details
i18n consistency check / check (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
Details
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
Details
web-search-agent tests / test (push) Canceled after 0s
Details
web-search-agent tests / agentbook (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
liqiang
2026-08-20 13:12:50 +00:00
commit b119135836
10275 changed files with 3284984 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/comparison.json
+700
View File
@@ -0,0 +1,700 @@
{
"experiment_id": "2-4",
"created_at": "2026-07-30T02:21:25.575202+08:00",
"protocol_sha256": "ebedaecdbb6c4fc8d8d92222dc17334a27a1b1b9f19e4732578563f5e9389424",
"protocol_copy": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/experiment_protocol.json",
"provider": "openai",
"model": "gpt-4o-mini",
"user_model": "gpt-4o-mini",
"objective_scoring": "vendored tau-bench environment reward",
"arms": {
"baseline": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-baseline_0730022052.json",
"sha256": "0edf8d281ea5e5d39ca5079d445b1684b1ec8de1026013590d68c5a0e88121fd"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
},
"tone_trump": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_trump_0730022059.json",
"sha256": "1531727c765cd3b1caef4b3c486399467979a40f951c603ae577241ed7918ccb"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
},
"tone_casual": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_0730022104.json",
"sha256": "4a1205d5ee3394b18e95d184a2f6dc7a565a2c1ba1558428b83f394f3d989b51"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
},
"wiki_random": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-wiki_random_0730022109.json",
"sha256": "bca0c69faf8faceabf444cc5f85c51bd7b4db4b72ea83151ed1ea88e60891264"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
},
"no_tool_desc": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-no_tool_desc_0730022115.json",
"sha256": "7a0fd841f28b3c6a8fe0f801bb33fba08acbdaa3a3dadbc0330334d209592ebf"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
},
"all_ablations": {
"artifact": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_wiki_random_no_tool_desc_0730022120.json",
"sha256": "aee798ceabad4e2022adcbadef6aca739937be4fbeffc9f98c9c65950bbd3990"
},
"rewards": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"success_rate": 0.0,
"total": 10,
"successes": 0,
"failures": 10,
"tasks_completed": 10,
"expected_tasks": 10,
"task_errors": [
{
"task_id": 0,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 1,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 2,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 3,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 4,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 5,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 6,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 7,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 8,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
},
{
"task_id": 9,
"error": "litellm.RateLimitError: RateLimitError: OpenAIException - You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."
}
],
"agent_steps": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_calls": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"tool_errors": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
],
"real_api_calls": 0,
"response_ids_present": false,
"usage": {
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
},
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"arm_complete": false
}
},
"credential_scan_passed": true,
"credential_findings": [],
"usage_and_cost": {
"total_real_api_calls": 0,
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0,
"observed_litellm_cost_usd": 0,
"all_calls_priced": true,
"qualification": "provider usage with LiteLLM response-cost estimate"
},
"campaign_complete": false,
"hypothesis_results": {
"historical_percentages_reproduced": false,
"qualification": "Current fixed ten-task campaign only; compare arm metrics directly."
}
}
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/experiment_protocol.json
+72
View File
@@ -0,0 +1,72 @@
{
"experiment_id": "2-4",
"protocol_version": "1.0.0",
"frozen_on": "2026-07-30",
"authority": "book/chapter2.md:654",
"benchmark": "vendored tau-bench airline test split",
"provider": "OpenAI",
"model": "gpt-4o-mini",
"user_model": "gpt-4o-mini",
"temperature": 0,
"user_temperature": 0,
"seed": 20260730,
"task_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
"trials_per_task": 1,
"max_agent_steps": 30,
"arms": {
"baseline": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_trump": {
"tone": "exaggerated Trump-style",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"tone_casual": {
"tone": "casual with emoji/slang",
"wiki": "original structured rules",
"tool_descriptions": "complete"
},
"wiki_random": {
"tone": "professional neutral",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "complete"
},
"no_tool_desc": {
"tone": "professional neutral",
"wiki": "original structured rules",
"tool_descriptions": "all description fields blank"
},
"all_ablations": {
"tone": "casual with emoji/slang",
"wiki": "same rules flattened and pre-shuffled",
"tool_descriptions": "all description fields blank"
}
},
"metrics": [
"tau-bench objective reward/pass rate",
"agent steps and model calls",
"tool calls and environment tool errors",
"provider prompt/completion/total tokens",
"provider response IDs",
"LiteLLM native cost estimate"
],
"acceptance_gates": [
"all six arms run the same ten task IDs with the same seed and user simulator",
"each task is scored only by the vendored tau-bench objective environment",
"every agent and simulated-user model call is a real provider response with ID and usage",
"raw request messages, tools, response content/tool calls, usage, timings, and trajectories are retained",
"the randomized wiki contains the same experimental rule material in the frozen pre-generated order",
"the no-description arm preserves schemas while blanking every nested description",
"campaign completion is independent of whether the manuscript hypotheses win",
"historical percentage claims are not treated as reproduced unless this fixed campaign independently yields them",
"credential scan passes"
],
"hypotheses": {
"tone": "tone variants have limited effect on objective task success",
"organization": "randomizing instruction organization reduces success",
"tool_descriptions": "removing descriptions increases tool errors and reduces success"
}
}
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/manifest.json
+33
View File
@@ -0,0 +1,33 @@
{
"experiment_id": "2-4",
"campaign_complete": false,
"protocol_sha256": "ebedaecdbb6c4fc8d8d92222dc17334a27a1b1b9f19e4732578563f5e9389424",
"summary_path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/comparison.json",
"summary_sha256": "0b26332ceba12fc3ab3e5713c4239a382607dd86b76d2e364dc1863067bad19a",
"arm_artifacts": {
"baseline": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-baseline_0730022052.json",
"sha256": "0edf8d281ea5e5d39ca5079d445b1684b1ec8de1026013590d68c5a0e88121fd"
},
"tone_trump": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_trump_0730022059.json",
"sha256": "1531727c765cd3b1caef4b3c486399467979a40f951c603ae577241ed7918ccb"
},
"tone_casual": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_0730022104.json",
"sha256": "4a1205d5ee3394b18e95d184a2f6dc7a565a2c1ba1558428b83f394f3d989b51"
},
"wiki_random": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-wiki_random_0730022109.json",
"sha256": "bca0c69faf8faceabf444cc5f85c51bd7b4db4b72ea83151ed1ea88e60891264"
},
"no_tool_desc": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-no_tool_desc_0730022115.json",
"sha256": "7a0fd841f28b3c6a8fe0f801bb33fba08acbdaa3a3dadbc0330334d209592ebf"
},
"all_ablations": {
"path": "/Users/boj/book/ai-agent-book/chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_wiki_random_no_tool_desc_0730022120.json",
"sha256": "aee798ceabad4e2022adcbadef6aca739937be4fbeffc9f98c9c65950bbd3990"
}
}
}
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-baseline_0730022052.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-no_tool_desc_0730022115.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_0730022104.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_casual_wiki_random_no_tool_desc_0730022120.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-tone_trump_0730022059.json
+142
View File
File diff suppressed because one or more lines are too long
chapter2/prompt-engineering/runs/exp2-4-gpt-4o-mini-20260730-v1/tool-calling-gpt-4o-mini-wiki_random_0730022109.json
+142
View File
File diff suppressed because one or more lines are too long