ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
This commit is contained in:
+302
@@ -0,0 +1,302 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "3-1-and-3-2",
|
||||
"run_id": "20260729T181741Z-3_1_and_3_2-8ab6407f",
|
||||
"provenance": {
|
||||
"captured_at": "2026-07-29T18:17:41.204547+00:00",
|
||||
"git_revision": "4a7f37cf278bd15948c409f14533017c4c7fbc29",
|
||||
"python": "3.11.4 (main, Jul 5 2023, 08:40:20) [Clang 14.0.6 ]",
|
||||
"platform": "macOS-26.3-arm64-arm-64bit",
|
||||
"credential_presence": {
|
||||
"ARK_API_KEY": true,
|
||||
"MOONSHOT_API_KEY": true,
|
||||
"OPENAI_API_KEY": true,
|
||||
"GEMINI_API_KEY": true,
|
||||
"SILICONFLOW_API_KEY": true
|
||||
}
|
||||
},
|
||||
"status": "partial",
|
||||
"scope": {
|
||||
"dataset_cases_available": 60,
|
||||
"cases_run": 1,
|
||||
"modes": [
|
||||
"notes"
|
||||
],
|
||||
"evaluations_completed": 1,
|
||||
"evaluations_expected": 1,
|
||||
"layers": [
|
||||
"layer1"
|
||||
]
|
||||
},
|
||||
"configuration": {
|
||||
"writer_provider": "ark",
|
||||
"writer_endpoint": "https://ark.cn-beijing.volces.com/api/v3",
|
||||
"writer_model": "doubao-seed-1-6-250615",
|
||||
"writer_seed": 37,
|
||||
"judge_provider": "moonshot",
|
||||
"judge_endpoint": "https://api.moonshot.cn/v1",
|
||||
"judge_model": "moonshot-v1-32k",
|
||||
"judge_is_external_to_writer": true,
|
||||
"workers": 1
|
||||
},
|
||||
"acceptance": {
|
||||
"all_60_cases": false,
|
||||
"twenty_per_layer": false,
|
||||
"all_four_modes": false,
|
||||
"sequential_memory_only": true,
|
||||
"independent_llm_judge": true,
|
||||
"all_calls_succeeded": true,
|
||||
"passed": false
|
||||
},
|
||||
"summary": {
|
||||
"aggregate": {
|
||||
"notes": {
|
||||
"layer1": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"token_usage": {
|
||||
"prompt_tokens": 5842,
|
||||
"completion_tokens": 2270,
|
||||
"total_tokens": 8112
|
||||
},
|
||||
"api_calls": 3,
|
||||
"errors": 0
|
||||
},
|
||||
"errors": [],
|
||||
"results": [
|
||||
{
|
||||
"test_id": "layer1_01_bank_account",
|
||||
"layer": "layer1",
|
||||
"title": "Bank Account Setup - Personal Details Retrieval",
|
||||
"mode": "notes",
|
||||
"session_count": 1,
|
||||
"memory_states": [
|
||||
{
|
||||
"session_index": 1,
|
||||
"conversation_id": "bank_setup_001",
|
||||
"memory": [
|
||||
{
|
||||
"conversation_id": "bank_setup_001"
|
||||
},
|
||||
{
|
||||
"timestamp": "2024-11-15 10:30:00"
|
||||
},
|
||||
{
|
||||
"business": "First National Bank"
|
||||
},
|
||||
{
|
||||
"department": "New Accounts"
|
||||
},
|
||||
{
|
||||
"call_duration": "47 minutes"
|
||||
},
|
||||
{
|
||||
"customer_full_name": "Michael James Robertson"
|
||||
},
|
||||
{
|
||||
"customer_date_of_birth": "March 15, 1985"
|
||||
},
|
||||
{
|
||||
"customer_residential_address": "1847 Maple Street, Apartment 3B, Portland, Oregon, 97205"
|
||||
},
|
||||
{
|
||||
"customer_address_duration": "about two and a half years"
|
||||
},
|
||||
{
|
||||
"customer_phone_number": "503-555-8924"
|
||||
},
|
||||
{
|
||||
"customer_email": "mrobertson85@email.com"
|
||||
},
|
||||
{
|
||||
"customer_employment_status": "employed full-time"
|
||||
},
|
||||
{
|
||||
"customer_occupation": "software engineer"
|
||||
},
|
||||
{
|
||||
"customer_employer": "TechCorp Industries"
|
||||
},
|
||||
{
|
||||
"customer_employment_duration": "coming up on four years this December"
|
||||
},
|
||||
{
|
||||
"customer_annual_income": "$125,000 a year, before taxes"
|
||||
},
|
||||
{
|
||||
"customer_social_security_number": "547-82-9163"
|
||||
},
|
||||
{
|
||||
"customer_driver_license": "Oregon license, number D758392"
|
||||
},
|
||||
{
|
||||
"checking_account_type": "Premium Checking"
|
||||
},
|
||||
{
|
||||
"Premium_Checking_minimum_balance": "$2,500"
|
||||
},
|
||||
{
|
||||
"savings_account_type": "Basic Savings"
|
||||
},
|
||||
{
|
||||
"Basic_Savings_minimum_balance": "$100"
|
||||
},
|
||||
{
|
||||
"checking_account_number": "4429853327"
|
||||
},
|
||||
{
|
||||
"savings_account_number": "4429853328"
|
||||
},
|
||||
{
|
||||
"online_banking_username": "MRobertson503"
|
||||
},
|
||||
{
|
||||
"mobile_banking_enrolled": "yes"
|
||||
},
|
||||
{
|
||||
"debit_card_requested": "yes"
|
||||
},
|
||||
{
|
||||
"debit_card_pin": "4827"
|
||||
},
|
||||
{
|
||||
"overdraft_protection": "linked to savings account"
|
||||
},
|
||||
{
|
||||
"initial_checking_deposit": "$5,000"
|
||||
},
|
||||
{
|
||||
"initial_savings_deposit": "$500"
|
||||
},
|
||||
{
|
||||
"deposit_method": "electronic transfer from Wells Fargo"
|
||||
},
|
||||
{
|
||||
"Wells_Fargo_account_number": "8847293001"
|
||||
},
|
||||
{
|
||||
"Wells_Fargo_routing_number": "121000248"
|
||||
},
|
||||
{
|
||||
"transfer_completion_time": "within 2-3 business days"
|
||||
},
|
||||
{
|
||||
"automatic_transfer_amount": "$200 monthly from checking to savings"
|
||||
},
|
||||
{
|
||||
"automatic_transfer_day": "15th of each month"
|
||||
},
|
||||
{
|
||||
"direct_deposit_setup": "paycheck from TechCorp Industries"
|
||||
},
|
||||
{
|
||||
"First_National_Bank_routing_number": "123006800"
|
||||
},
|
||||
{
|
||||
"debit_card_international_use": "yes"
|
||||
},
|
||||
{
|
||||
"debit_card_foreign_transaction_fee": "3%"
|
||||
},
|
||||
{
|
||||
"travel_notifications_status": "not set up"
|
||||
},
|
||||
{
|
||||
"security_question_1_answer": "Buddy (first pet name)"
|
||||
},
|
||||
{
|
||||
"security_question_2_answer": "Denver, Colorado (city born in)"
|
||||
},
|
||||
{
|
||||
"security_question_3_answer": "Harrison (mother's maiden name)"
|
||||
},
|
||||
{
|
||||
"statements_preference": "electronic statements"
|
||||
},
|
||||
{
|
||||
"statements_email": "mrobertson85@email.com"
|
||||
},
|
||||
{
|
||||
"credit_card_application_status": "not applied for, will consider later"
|
||||
},
|
||||
{
|
||||
"debit_card_arrival_time": "7-10 business days"
|
||||
},
|
||||
{
|
||||
"debit_card_delivery_address": "1847 Maple Street, Apartment 3B, Portland, Oregon, 97205"
|
||||
}
|
||||
],
|
||||
"isolation": {
|
||||
"prior_raw_histories_supplied": 0,
|
||||
"current_memory_supplied": true,
|
||||
"new_history_supplied": "bank_setup_001"
|
||||
}
|
||||
}
|
||||
],
|
||||
"answer": "Your checking account number is 4429853327. For direct deposit setup, you may also need the bank's routing number, which is 123006800. Let me know if you need further details!",
|
||||
"judge": {
|
||||
"scores": {
|
||||
"precision": 4,
|
||||
"recall": 4,
|
||||
"reasoning": 4,
|
||||
"proactivity": 4
|
||||
},
|
||||
"hallucination_veto": false,
|
||||
"passed": true,
|
||||
"reward": 1.0
|
||||
},
|
||||
"judge_raw": {
|
||||
"dimensions": {
|
||||
"precision": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent accurately provided the specific checking account number 4429853327 that was requested by the user.",
|
||||
"evidence": [
|
||||
"USER: What was my checking account number again? I need it to set up my direct deposit at work.",
|
||||
"ANSWER: Your checking account number is 4429853327."
|
||||
]
|
||||
},
|
||||
"recall": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent recalled the correct checking account number from the conversation without any confusion with other numbers.",
|
||||
"evidence": [
|
||||
"ASSISTANT: Correct. And your savings account number is 4429853328.",
|
||||
"ANSWER: Your checking account number is 4429853327."
|
||||
]
|
||||
},
|
||||
"reasoning": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent demonstrated proper reasoning by directly addressing the user's request and providing the exact checking account number without any unnecessary information.",
|
||||
"evidence": [
|
||||
"USER: What was my checking account number again? I need it to set up my direct deposit at work.",
|
||||
"ANSWER: Your checking account number is 4429853327."
|
||||
]
|
||||
},
|
||||
"proactivity": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent not only provided the checking account number but also proactively mentioned the routing number, which is relevant for setting up direct deposit.",
|
||||
"evidence": [
|
||||
"ANSWER: Your checking account number is 4429853327. For direct deposit setup, you may also need the bank's routing number, which is 123006800."
|
||||
]
|
||||
}
|
||||
},
|
||||
"hallucination": {
|
||||
"detected": false,
|
||||
"claims": [],
|
||||
"reasoning": "The agent's response was based on the information provided in the conversation and did not include any unsupported or contradicted factual claims."
|
||||
},
|
||||
"overall_reasoning": "The agent's response was exemplary in terms of precision, recall, reasoning, and proactivity. It accurately provided the specific checking account number requested by the user and proactively included the routing number, which is necessary for setting up direct deposit. The response was concise, relevant, and demonstrated a clear understanding of the user's needs."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "3-1-and-3-2",
|
||||
"run_id": "20260729T181741Z-3_1_and_3_2-8ab6407f",
|
||||
"created_at": "2026-07-29T18:17:41.206311+00:00",
|
||||
"status": "partial",
|
||||
"run_dir": "/Users/boj/book/ai-agent-book/chapter3/user-memory/validation/runs/20260729T181741Z-3_1_and_3_2-8ab6407f",
|
||||
"artifacts": {
|
||||
"evidence.json": "cd522d7a6c46d2f02b69d479146cc8470b41b1a447cfc9eeb538a49dbdb71e43",
|
||||
"receipts.json": "99c2430da27f37fbf9f93d8ce35798d52cfc7c7a95c3e7b9ffb1612b4b714361"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/01_bank_account_setup.yaml",
|
||||
"sha256": "e9f788fa393a0f07c83f32e90f05d0b2212355ba62e0a5c3288a469b5feb75b6",
|
||||
"bytes": 11396
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"aggregate": {
|
||||
"notes": {
|
||||
"layer1": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"token_usage": {
|
||||
"prompt_tokens": 5842,
|
||||
"completion_tokens": 2270,
|
||||
"total_tokens": 8112
|
||||
},
|
||||
"api_calls": 3,
|
||||
"errors": 0
|
||||
},
|
||||
"acceptance": {
|
||||
"all_60_cases": false,
|
||||
"twenty_per_layer": false,
|
||||
"all_four_modes": false,
|
||||
"sequential_memory_only": true,
|
||||
"independent_llm_judge": true,
|
||||
"all_calls_succeeded": true,
|
||||
"passed": false
|
||||
}
|
||||
}
|
||||
+230
File diff suppressed because one or more lines are too long
+336
@@ -0,0 +1,336 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "3-1-and-3-2",
|
||||
"run_id": "20260729T190850Z-3_1_and_3_2-1f76f65d",
|
||||
"provenance": {
|
||||
"captured_at": "2026-07-29T19:08:50.076181+00:00",
|
||||
"git_revision": "4a7f37cf278bd15948c409f14533017c4c7fbc29",
|
||||
"python": "3.11.4 (main, Jul 5 2023, 08:40:20) [Clang 14.0.6 ]",
|
||||
"platform": "macOS-26.3-arm64-arm-64bit",
|
||||
"credential_presence": {
|
||||
"ARK_API_KEY": true,
|
||||
"MOONSHOT_API_KEY": true,
|
||||
"OPENAI_API_KEY": true,
|
||||
"GEMINI_API_KEY": true,
|
||||
"SILICONFLOW_API_KEY": true
|
||||
}
|
||||
},
|
||||
"status": "partial",
|
||||
"scope": {
|
||||
"dataset_cases_available": 60,
|
||||
"cases_run": 1,
|
||||
"modes": [
|
||||
"notes"
|
||||
],
|
||||
"evaluations_completed": 1,
|
||||
"evaluations_expected": 1,
|
||||
"layers": [
|
||||
"layer1"
|
||||
]
|
||||
},
|
||||
"configuration": {
|
||||
"writer_provider": "ark",
|
||||
"writer_endpoint": "https://ark.cn-beijing.volces.com/api/v3",
|
||||
"writer_model": "doubao-seed-1-6-250615",
|
||||
"writer_seed": 37,
|
||||
"judge_provider": "moonshot",
|
||||
"judge_endpoint": "https://api.moonshot.cn/v1",
|
||||
"judge_model": "moonshot-v1-32k",
|
||||
"judge_is_external_to_writer": true,
|
||||
"workers": 1
|
||||
},
|
||||
"acceptance": {
|
||||
"all_60_cases": false,
|
||||
"twenty_per_layer": false,
|
||||
"all_four_modes": false,
|
||||
"sequential_memory_only": true,
|
||||
"independent_llm_judge": true,
|
||||
"all_calls_succeeded": true,
|
||||
"passed": false
|
||||
},
|
||||
"summary": {
|
||||
"aggregate": {
|
||||
"notes": {
|
||||
"layer1": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"token_usage": {
|
||||
"prompt_tokens": 5882,
|
||||
"completion_tokens": 2215,
|
||||
"total_tokens": 8097
|
||||
},
|
||||
"by_mode": {
|
||||
"notes": {
|
||||
"api_calls": 3,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 5882,
|
||||
"completion_tokens": 2215,
|
||||
"total_tokens": 8097
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 42227.854,
|
||||
"mean_per_call": 14075.951333333333
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"advanced_json_cards": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"api_calls": 3,
|
||||
"errors": 0
|
||||
},
|
||||
"errors": [],
|
||||
"results": [
|
||||
{
|
||||
"test_id": "layer1_01_bank_account",
|
||||
"layer": "layer1",
|
||||
"title": "Bank Account Setup - Personal Details Retrieval",
|
||||
"mode": "notes",
|
||||
"session_count": 1,
|
||||
"memory_states": [
|
||||
{
|
||||
"session_index": 1,
|
||||
"conversation_id": "bank_setup_001",
|
||||
"memory": [
|
||||
{
|
||||
"conversation_id": "bank_setup_001"
|
||||
},
|
||||
{
|
||||
"timestamp": "2024-11-15 10:30:00"
|
||||
},
|
||||
{
|
||||
"business": "First National Bank"
|
||||
},
|
||||
{
|
||||
"department": "New Accounts"
|
||||
},
|
||||
{
|
||||
"call_duration": "47 minutes"
|
||||
},
|
||||
{
|
||||
"user_full_name": "Michael James Robertson"
|
||||
},
|
||||
{
|
||||
"user_date_of_birth": "March 15, 1985"
|
||||
},
|
||||
{
|
||||
"user_residential_address": "1847 Maple Street, Apartment 3B, Portland, Oregon, 97205"
|
||||
},
|
||||
{
|
||||
"user_length_at_address": "about two and a half years"
|
||||
},
|
||||
{
|
||||
"user_phone_number": "503-555-8924"
|
||||
},
|
||||
{
|
||||
"user_email": "mrobertson85@email.com"
|
||||
},
|
||||
{
|
||||
"user_employment_status": "employed full-time as a software engineer at TechCorp Industries"
|
||||
},
|
||||
{
|
||||
"user_employment_length_at_TechCorp": "coming up on four years in December"
|
||||
},
|
||||
{
|
||||
"user_annual_income": "$125,000 before taxes"
|
||||
},
|
||||
{
|
||||
"user_social_security_number": "547-82-9163"
|
||||
},
|
||||
{
|
||||
"user_driver_license": "Oregon, number D758392"
|
||||
},
|
||||
{
|
||||
"checking_account_type": "Premium Checking",
|
||||
"minimum_balance": "$2,500"
|
||||
},
|
||||
{
|
||||
"savings_account_type": "Basic Savings",
|
||||
"minimum_balance": "$100"
|
||||
},
|
||||
{
|
||||
"checking_account_number": "4429853327",
|
||||
"owner": "Michael James Robertson"
|
||||
},
|
||||
{
|
||||
"savings_account_number": "4429853328",
|
||||
"owner": "Michael James Robertson"
|
||||
},
|
||||
{
|
||||
"debit_card_requested": true,
|
||||
"pin": "4827"
|
||||
},
|
||||
{
|
||||
"online_banking_username": "MRobertson503"
|
||||
},
|
||||
{
|
||||
"mobile_banking_enrolled": true
|
||||
},
|
||||
{
|
||||
"initial_deposit_checking": "$5,000"
|
||||
},
|
||||
{
|
||||
"initial_deposit_savings": "$500"
|
||||
},
|
||||
{
|
||||
"initial_deposit_method": "electronic transfer from Wells Fargo"
|
||||
},
|
||||
{
|
||||
"wells_fargo_account_number": "8847293001"
|
||||
},
|
||||
{
|
||||
"wells_fargo_routing_number": "121000248"
|
||||
},
|
||||
{
|
||||
"transfer_initiated_date": "2024-11-15"
|
||||
},
|
||||
{
|
||||
"transfer_completion_time": "2-3 business days"
|
||||
},
|
||||
{
|
||||
"automatic_transfer_amount": "$200",
|
||||
"frequency": "monthly",
|
||||
"from_account": "checking",
|
||||
"to_account": "savings",
|
||||
"transfer_day": "15th of each month"
|
||||
},
|
||||
{
|
||||
"overdraft_protection_linked_to": "savings account"
|
||||
},
|
||||
{
|
||||
"debit_card_delivery_time": "7-10 business days"
|
||||
},
|
||||
{
|
||||
"debit_card_delivery_address": "1847 Maple Street, Apartment 3B, Portland, Oregon, 97205"
|
||||
},
|
||||
{
|
||||
"first_national_bank_routing_number": "123006800"
|
||||
},
|
||||
{
|
||||
"atm_benefit": "unlimited free withdrawals at First National Bank ATMs"
|
||||
},
|
||||
{
|
||||
"atm_benefit": "reimbursement up to $15 per month for other bank ATM fees"
|
||||
},
|
||||
{
|
||||
"first_national_bank_atms_in_portland": "47"
|
||||
},
|
||||
{
|
||||
"allpoint_atm_network_access": "free, over 55,000 ATMs nationwide"
|
||||
},
|
||||
{
|
||||
"security_question_1": "first pet name",
|
||||
"answer": "Buddy"
|
||||
},
|
||||
{
|
||||
"security_question_2": "birth city",
|
||||
"answer": "Denver, Colorado"
|
||||
},
|
||||
{
|
||||
"security_question_3": "mother's maiden name",
|
||||
"answer": "Harrison"
|
||||
},
|
||||
{
|
||||
"statement_preference": "electronic",
|
||||
"email": "mrobertson85@email.com"
|
||||
}
|
||||
],
|
||||
"isolation": {
|
||||
"prior_raw_histories_supplied": 0,
|
||||
"current_memory_supplied": true,
|
||||
"new_history_supplied": "bank_setup_001"
|
||||
}
|
||||
}
|
||||
],
|
||||
"answer": "Your checking account number is **4429853327**. For direct deposit setup, you may also need the bank's routing number, which is **123006800**. Let me know if you need any other details!",
|
||||
"judge": {
|
||||
"scores": {
|
||||
"precision": 4,
|
||||
"recall": 4,
|
||||
"reasoning": 4,
|
||||
"proactivity": 4
|
||||
},
|
||||
"hallucination_veto": false,
|
||||
"passed": true,
|
||||
"reward": 1.0
|
||||
},
|
||||
"judge_raw": {
|
||||
"dimensions": {
|
||||
"precision": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent accurately retrieved and provided the specific checking account number 4429853327 that was mentioned during the account setup conversation.",
|
||||
"evidence": [
|
||||
"USER: What was my checking account number again? I need it to set up my direct deposit at work.",
|
||||
"ANSWER: Your checking account number is **4429853327**."
|
||||
]
|
||||
},
|
||||
"recall": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent not only recalled the checking account number but also proactively provided the routing number, which is relevant for setting up direct deposit.",
|
||||
"evidence": [
|
||||
"ANSWER: Your checking account number is **4429853327**. For direct deposit setup, you may also need the bank's routing number, which is **123006800**."
|
||||
]
|
||||
},
|
||||
"reasoning": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent demonstrated proper memory recall and understanding of the context by providing the exact checking account number without confusion with other numbers mentioned in the conversation."
|
||||
},
|
||||
"proactivity": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent went beyond the user's request by also mentioning the routing number, which is necessary for direct deposit setup, showing a proactive approach to assist the user."
|
||||
}
|
||||
},
|
||||
"hallucination": {
|
||||
"detected": false,
|
||||
"claims": [],
|
||||
"reasoning": "The agent's response is based on the information provided in the conversation and does not include any unsupported or contradicted factual claims."
|
||||
},
|
||||
"overall_reasoning": "The agent's response is exemplary as it accurately provides the requested checking account number and goes a step further by including the routing number, which is essential for the user's direct deposit setup. This demonstrates both precision in recalling the specific account number and proactivity in anticipating the user's needs."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "3-1-and-3-2",
|
||||
"run_id": "20260729T190850Z-3_1_and_3_2-1f76f65d",
|
||||
"created_at": "2026-07-29T19:08:50.077877+00:00",
|
||||
"status": "partial",
|
||||
"run_dir": "/Users/boj/book/ai-agent-book/chapter3/user-memory/validation/runs/20260729T190850Z-3_1_and_3_2-1f76f65d",
|
||||
"artifacts": {
|
||||
"evidence.json": "295ecda38ea1a38c3416de27b9fb4b0c1593990005a48684ae947f2118aa9e8c",
|
||||
"receipts.json": "51ab7fffdc98c4c2a0d1afc5ae9f08c67de6aaf1ad6388c6e4c00cb135171b2c"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/01_bank_account_setup.yaml",
|
||||
"sha256": "e9f788fa393a0f07c83f32e90f05d0b2212355ba62e0a5c3288a469b5feb75b6",
|
||||
"bytes": 11396
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"aggregate": {
|
||||
"notes": {
|
||||
"layer1": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"token_usage": {
|
||||
"prompt_tokens": 5882,
|
||||
"completion_tokens": 2215,
|
||||
"total_tokens": 8097
|
||||
},
|
||||
"by_mode": {
|
||||
"notes": {
|
||||
"api_calls": 3,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 5882,
|
||||
"completion_tokens": 2215,
|
||||
"total_tokens": 8097
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 42227.854,
|
||||
"mean_per_call": 14075.951333333333
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"advanced_json_cards": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"api_calls": 3,
|
||||
"errors": 0
|
||||
},
|
||||
"acceptance": {
|
||||
"all_60_cases": false,
|
||||
"twenty_per_layer": false,
|
||||
"all_four_modes": false,
|
||||
"sequential_memory_only": true,
|
||||
"independent_llm_judge": true,
|
||||
"all_calls_succeeded": true,
|
||||
"passed": false
|
||||
}
|
||||
}
|
||||
+231
File diff suppressed because one or more lines are too long
+336
@@ -0,0 +1,336 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "3-1-and-3-2",
|
||||
"run_id": "20260729T190859Z-3_1_and_3_2-13c7ccd7",
|
||||
"provenance": {
|
||||
"captured_at": "2026-07-29T19:08:59.390853+00:00",
|
||||
"git_revision": "4a7f37cf278bd15948c409f14533017c4c7fbc29",
|
||||
"python": "3.11.4 (main, Jul 5 2023, 08:40:20) [Clang 14.0.6 ]",
|
||||
"platform": "macOS-26.3-arm64-arm-64bit",
|
||||
"credential_presence": {
|
||||
"ARK_API_KEY": true,
|
||||
"MOONSHOT_API_KEY": true,
|
||||
"OPENAI_API_KEY": true,
|
||||
"GEMINI_API_KEY": true,
|
||||
"SILICONFLOW_API_KEY": true
|
||||
}
|
||||
},
|
||||
"status": "partial",
|
||||
"scope": {
|
||||
"dataset_cases_available": 60,
|
||||
"cases_run": 1,
|
||||
"modes": [
|
||||
"notes"
|
||||
],
|
||||
"evaluations_completed": 1,
|
||||
"evaluations_expected": 1,
|
||||
"layers": [
|
||||
"layer1"
|
||||
]
|
||||
},
|
||||
"configuration": {
|
||||
"writer_provider": "ark",
|
||||
"writer_endpoint": "https://ark.cn-beijing.volces.com/api/v3",
|
||||
"writer_model": "doubao-seed-1-6-250615",
|
||||
"writer_seed": 37,
|
||||
"judge_provider": "moonshot",
|
||||
"judge_endpoint": "https://api.moonshot.cn/v1",
|
||||
"judge_model": "moonshot-v1-32k",
|
||||
"judge_is_external_to_writer": true,
|
||||
"workers": 1
|
||||
},
|
||||
"acceptance": {
|
||||
"all_60_cases": false,
|
||||
"twenty_per_layer": false,
|
||||
"all_four_modes": false,
|
||||
"sequential_memory_only": true,
|
||||
"independent_llm_judge": true,
|
||||
"all_calls_succeeded": true,
|
||||
"passed": false
|
||||
},
|
||||
"summary": {
|
||||
"aggregate": {
|
||||
"notes": {
|
||||
"layer1": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"token_usage": {
|
||||
"prompt_tokens": 5882,
|
||||
"completion_tokens": 2215,
|
||||
"total_tokens": 8097
|
||||
},
|
||||
"by_mode": {
|
||||
"notes": {
|
||||
"api_calls": 3,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 5882,
|
||||
"completion_tokens": 2215,
|
||||
"total_tokens": 8097
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 42227.854,
|
||||
"mean_per_call": 14075.951333333333
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"advanced_json_cards": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"api_calls": 3,
|
||||
"errors": 0
|
||||
},
|
||||
"errors": [],
|
||||
"results": [
|
||||
{
|
||||
"test_id": "layer1_01_bank_account",
|
||||
"layer": "layer1",
|
||||
"title": "Bank Account Setup - Personal Details Retrieval",
|
||||
"mode": "notes",
|
||||
"session_count": 1,
|
||||
"memory_states": [
|
||||
{
|
||||
"session_index": 1,
|
||||
"conversation_id": "bank_setup_001",
|
||||
"memory": [
|
||||
{
|
||||
"conversation_id": "bank_setup_001"
|
||||
},
|
||||
{
|
||||
"timestamp": "2024-11-15 10:30:00"
|
||||
},
|
||||
{
|
||||
"business": "First National Bank"
|
||||
},
|
||||
{
|
||||
"department": "New Accounts"
|
||||
},
|
||||
{
|
||||
"call_duration": "47 minutes"
|
||||
},
|
||||
{
|
||||
"user_full_name": "Michael James Robertson"
|
||||
},
|
||||
{
|
||||
"user_date_of_birth": "March 15, 1985"
|
||||
},
|
||||
{
|
||||
"user_residential_address": "1847 Maple Street, Apartment 3B, Portland, Oregon, 97205"
|
||||
},
|
||||
{
|
||||
"user_length_at_address": "about two and a half years"
|
||||
},
|
||||
{
|
||||
"user_phone_number": "503-555-8924"
|
||||
},
|
||||
{
|
||||
"user_email": "mrobertson85@email.com"
|
||||
},
|
||||
{
|
||||
"user_employment_status": "employed full-time as a software engineer at TechCorp Industries"
|
||||
},
|
||||
{
|
||||
"user_employment_length_at_TechCorp": "coming up on four years in December"
|
||||
},
|
||||
{
|
||||
"user_annual_income": "$125,000 before taxes"
|
||||
},
|
||||
{
|
||||
"user_social_security_number": "547-82-9163"
|
||||
},
|
||||
{
|
||||
"user_driver_license": "Oregon, number D758392"
|
||||
},
|
||||
{
|
||||
"checking_account_type": "Premium Checking",
|
||||
"minimum_balance": "$2,500"
|
||||
},
|
||||
{
|
||||
"savings_account_type": "Basic Savings",
|
||||
"minimum_balance": "$100"
|
||||
},
|
||||
{
|
||||
"checking_account_number": "4429853327",
|
||||
"owner": "Michael James Robertson"
|
||||
},
|
||||
{
|
||||
"savings_account_number": "4429853328",
|
||||
"owner": "Michael James Robertson"
|
||||
},
|
||||
{
|
||||
"debit_card_requested": true,
|
||||
"pin": "4827"
|
||||
},
|
||||
{
|
||||
"online_banking_username": "MRobertson503"
|
||||
},
|
||||
{
|
||||
"mobile_banking_enrolled": true
|
||||
},
|
||||
{
|
||||
"initial_deposit_checking": "$5,000"
|
||||
},
|
||||
{
|
||||
"initial_deposit_savings": "$500"
|
||||
},
|
||||
{
|
||||
"initial_deposit_method": "electronic transfer from Wells Fargo"
|
||||
},
|
||||
{
|
||||
"wells_fargo_account_number": "8847293001"
|
||||
},
|
||||
{
|
||||
"wells_fargo_routing_number": "121000248"
|
||||
},
|
||||
{
|
||||
"transfer_initiated_date": "2024-11-15"
|
||||
},
|
||||
{
|
||||
"transfer_completion_time": "2-3 business days"
|
||||
},
|
||||
{
|
||||
"automatic_transfer_amount": "$200",
|
||||
"frequency": "monthly",
|
||||
"from_account": "checking",
|
||||
"to_account": "savings",
|
||||
"transfer_day": "15th of each month"
|
||||
},
|
||||
{
|
||||
"overdraft_protection_linked_to": "savings account"
|
||||
},
|
||||
{
|
||||
"debit_card_delivery_time": "7-10 business days"
|
||||
},
|
||||
{
|
||||
"debit_card_delivery_address": "1847 Maple Street, Apartment 3B, Portland, Oregon, 97205"
|
||||
},
|
||||
{
|
||||
"first_national_bank_routing_number": "123006800"
|
||||
},
|
||||
{
|
||||
"atm_benefit": "unlimited free withdrawals at First National Bank ATMs"
|
||||
},
|
||||
{
|
||||
"atm_benefit": "reimbursement up to $15 per month for other bank ATM fees"
|
||||
},
|
||||
{
|
||||
"first_national_bank_atms_in_portland": "47"
|
||||
},
|
||||
{
|
||||
"allpoint_atm_network_access": "free, over 55,000 ATMs nationwide"
|
||||
},
|
||||
{
|
||||
"security_question_1": "first pet name",
|
||||
"answer": "Buddy"
|
||||
},
|
||||
{
|
||||
"security_question_2": "birth city",
|
||||
"answer": "Denver, Colorado"
|
||||
},
|
||||
{
|
||||
"security_question_3": "mother's maiden name",
|
||||
"answer": "Harrison"
|
||||
},
|
||||
{
|
||||
"statement_preference": "electronic",
|
||||
"email": "mrobertson85@email.com"
|
||||
}
|
||||
],
|
||||
"isolation": {
|
||||
"prior_raw_histories_supplied": 0,
|
||||
"current_memory_supplied": true,
|
||||
"new_history_supplied": "bank_setup_001"
|
||||
}
|
||||
}
|
||||
],
|
||||
"answer": "Your checking account number is **4429853327**. For direct deposit setup, you may also need the bank's routing number, which is **123006800**. Let me know if you need any other details!",
|
||||
"judge": {
|
||||
"scores": {
|
||||
"precision": 4,
|
||||
"recall": 4,
|
||||
"reasoning": 4,
|
||||
"proactivity": 4
|
||||
},
|
||||
"hallucination_veto": false,
|
||||
"passed": true,
|
||||
"reward": 1.0
|
||||
},
|
||||
"judge_raw": {
|
||||
"dimensions": {
|
||||
"precision": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent accurately retrieved and provided the specific checking account number 4429853327 that was mentioned during the account setup conversation.",
|
||||
"evidence": [
|
||||
"USER: What was my checking account number again? I need it to set up my direct deposit at work.",
|
||||
"ANSWER: Your checking account number is **4429853327**."
|
||||
]
|
||||
},
|
||||
"recall": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent not only recalled the checking account number but also proactively provided the routing number, which is relevant for setting up direct deposit.",
|
||||
"evidence": [
|
||||
"ANSWER: Your checking account number is **4429853327**. For direct deposit setup, you may also need the bank's routing number, which is **123006800**."
|
||||
]
|
||||
},
|
||||
"reasoning": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent demonstrated proper memory recall and understanding of the context by providing the exact checking account number without confusion with other numbers mentioned in the conversation."
|
||||
},
|
||||
"proactivity": {
|
||||
"score": 4,
|
||||
"reasoning": "The agent went beyond the user's request by also mentioning the routing number, which is necessary for direct deposit setup, showing a proactive approach to assist the user."
|
||||
}
|
||||
},
|
||||
"hallucination": {
|
||||
"detected": false,
|
||||
"claims": [],
|
||||
"reasoning": "The agent's response is based on the information provided in the conversation and does not include any unsupported or contradicted factual claims."
|
||||
},
|
||||
"overall_reasoning": "The agent's response is exemplary as it accurately provides the requested checking account number and goes a step further by including the routing number, which is essential for the user's direct deposit setup. This demonstrates both precision in recalling the specific account number and proactivity in anticipating the user's needs."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "3-1-and-3-2",
|
||||
"run_id": "20260729T190859Z-3_1_and_3_2-13c7ccd7",
|
||||
"created_at": "2026-07-29T19:08:59.404059+00:00",
|
||||
"status": "partial",
|
||||
"run_dir": "/Users/boj/book/ai-agent-book/chapter3/user-memory/validation/runs/20260729T190859Z-3_1_and_3_2-13c7ccd7",
|
||||
"artifacts": {
|
||||
"evidence.json": "3c8c78ab09346a90dbc70ac0fa88ffdcdaaacb3023b42257b915dbbfa4f895f7",
|
||||
"receipts.json": "51ab7fffdc98c4c2a0d1afc5ae9f08c67de6aaf1ad6388c6e4c00cb135171b2c"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/01_bank_account_setup.yaml",
|
||||
"sha256": "e9f788fa393a0f07c83f32e90f05d0b2212355ba62e0a5c3288a469b5feb75b6",
|
||||
"bytes": 11396
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"aggregate": {
|
||||
"notes": {
|
||||
"layer1": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 1,
|
||||
"pass_rate": 1.0,
|
||||
"mean_reward": 1.0,
|
||||
"hallucination_rate": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"token_usage": {
|
||||
"prompt_tokens": 5882,
|
||||
"completion_tokens": 2215,
|
||||
"total_tokens": 8097
|
||||
},
|
||||
"by_mode": {
|
||||
"notes": {
|
||||
"api_calls": 3,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 5882,
|
||||
"completion_tokens": 2215,
|
||||
"total_tokens": 8097
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 42227.854,
|
||||
"mean_per_call": 14075.951333333333
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"advanced_json_cards": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"api_calls": 3,
|
||||
"errors": 0
|
||||
},
|
||||
"acceptance": {
|
||||
"all_60_cases": false,
|
||||
"twenty_per_layer": false,
|
||||
"all_four_modes": false,
|
||||
"sequential_memory_only": true,
|
||||
"independent_llm_judge": true,
|
||||
"all_calls_succeeded": true,
|
||||
"passed": false
|
||||
}
|
||||
}
|
||||
+231
File diff suppressed because one or more lines are too long
+38419
File diff suppressed because it is too large
Load Diff
+410
@@ -0,0 +1,410 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "3-1-and-3-2",
|
||||
"run_id": "20260729T210458Z-3_1_and_3_2-0fba882a",
|
||||
"created_at": "2026-07-29T21:04:58.442322+00:00",
|
||||
"status": "partial",
|
||||
"run_dir": "/Users/boj/book/ai-agent-book/chapter3/user-memory/validation/runs/20260729T210458Z-3_1_and_3_2-0fba882a",
|
||||
"artifacts": {
|
||||
"evidence.json": "36c9f84f190dd4b4656e70382735468dcddf62a771251f2305a1a3c37d515071",
|
||||
"receipts.json": "89056c3bc4a4856b51b15f6a8443955cbcd50904629653d09ba70992e0db13e1"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/01_bank_account_setup.yaml",
|
||||
"sha256": "e9f788fa393a0f07c83f32e90f05d0b2212355ba62e0a5c3288a469b5feb75b6",
|
||||
"bytes": 11396
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/02_insurance_claim.yaml",
|
||||
"sha256": "ba4d78069e6aaedc1876cbbd3069250ebeafd1def4b619463218ad3a33f80119",
|
||||
"bytes": 13222
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/03_medical_appointment.yaml",
|
||||
"sha256": "42877892ab8720090853ac0fa9050e4097df967a4d88893e25563766e536d3bb",
|
||||
"bytes": 12942
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/04_airline_booking.yaml",
|
||||
"sha256": "76b354c62739d5a23cc71fd94c7a16a0c9552fe8014bc9ddafe262ea0df366de",
|
||||
"bytes": 11836
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/05_internet_service_setup.yaml",
|
||||
"sha256": "f5836b44ae54ee6c220b0261fce73f27e211ed5436b8a52afad17ba6c399658d",
|
||||
"bytes": 12283
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/06_credit_card_application.yaml",
|
||||
"sha256": "7c4358326df8671cad9814a8796df48e5a60cd807ee89a33e6d8523302e8630e",
|
||||
"bytes": 11172
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/07_car_rental.yaml",
|
||||
"sha256": "e497e5819e228f25c58a6f40a29d9d56931eef19c032480ee09eb247a483ebea",
|
||||
"bytes": 10705
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/08_hotel_reservation.yaml",
|
||||
"sha256": "16ddb69e00d880376e869fa454af0b697301c24d7b6b0be20e090ebd9ef64c20",
|
||||
"bytes": 10229
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/09_home_security.yaml",
|
||||
"sha256": "50bc194260c94f8f4e0dd773e5a07dc58d0bf47b72df40aa4a1bc2f7adf5ab3c",
|
||||
"bytes": 11064
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/10_pharmacy_transfer.yaml",
|
||||
"sha256": "d7051d7da1e99b8c662c02cf4bf8a790e72c82efc684562daeb6714f6cd67279",
|
||||
"bytes": 9554
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/11_mortgage_application.yaml",
|
||||
"sha256": "b34269d9b1adc69dc1579317215e5e410c3f85f9f688b5ac283bd419df91a006",
|
||||
"bytes": 14009
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/12_gym_membership.yaml",
|
||||
"sha256": "c530e2215951be9f6553c3ee0c29e68ea596bd9215fa72deda13a44da8ff4d79",
|
||||
"bytes": 12243
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/13_tax_preparation.yaml",
|
||||
"sha256": "ce2af37f2ace66b65cd0f834b8fa727abd7f5399ae260c43c26d07f00bd20e03",
|
||||
"bytes": 14382
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/14_cellphone_upgrade.yaml",
|
||||
"sha256": "26fbf77437deaec9d9bfc635700d3a9f688904d8db594d47b337a2db3de25d83",
|
||||
"bytes": 11402
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/15_college_enrollment.yaml",
|
||||
"sha256": "4252cded12f1e5821d3ad3cbbe270dc4c526cf6ffb86e53bea6db163a3e4f847",
|
||||
"bytes": 12507
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/16_home_renovation.yaml",
|
||||
"sha256": "e66f44d9b3d632cf2afc2759386e8e81dc0edb89b735450fcf5e2e54e2db24fc",
|
||||
"bytes": 11383
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/17_veterinary_care.yaml",
|
||||
"sha256": "712b5726f9d3732d915cb338bf7080d5c048f1156e0718434befa2b86ae92013",
|
||||
"bytes": 10538
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/18_retirement_planning.yaml",
|
||||
"sha256": "46e3876a7d84c70b2bba668f2454991d15066628e83cce97e5fb6abd2ba6558f",
|
||||
"bytes": 10650
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/19_wedding_venue.yaml",
|
||||
"sha256": "b2066bf3897c9b022b63321d9499a254d4b709069b1f5e5d9a0a69dc44026493",
|
||||
"bytes": 10141
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/20_daycare_enrollment.yaml",
|
||||
"sha256": "d6d6a8e6c1130fd693f7aa2febc8e40b49df4848f2ed45ccc849dc78432f315e",
|
||||
"bytes": 11030
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/01_multiple_vehicles.yaml",
|
||||
"sha256": "d00b83e6db7660a2686606218e6b592af883aea201b066a3f6e769d7710e4b48",
|
||||
"bytes": 19514
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/02_multiple_properties.yaml",
|
||||
"sha256": "21b6b80b63eb36481c12317ed63be09d033bcaf7e2526e8abf91d2bdd70d8451",
|
||||
"bytes": 19694
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/03_multiple_credit_cards.yaml",
|
||||
"sha256": "3cde2ffbf8f4f75ed4fc3e1f59c0a1c388582f3223a792e1f496b5ba1b865486",
|
||||
"bytes": 23868
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/04_multiple_subscriptions.yaml",
|
||||
"sha256": "25e2dab7c6f1e12a9ce72646dd702026b19b15d1c8a76aec5d82be4eb0503298",
|
||||
"bytes": 22110
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/05_multiple_bank_accounts.yaml",
|
||||
"sha256": "b30b4c9e503737bd600957e1f3783d046930263ecb78c970056e63b25ee63cc6",
|
||||
"bytes": 21604
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/06_multiple_insurance_policies.yaml",
|
||||
"sha256": "28ec764bd484bb7856941168dc7e557c4eaa791fe54e6b0e466dd6b0ab7776ea",
|
||||
"bytes": 22244
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/07_multiple_medications.yaml",
|
||||
"sha256": "abaa76f5e90ddf3f1a1af1e5bb75fba0b3fae3d77fceb86c54da0d2584c50c24",
|
||||
"bytes": 27025
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/08_multiple_rental_properties.yaml",
|
||||
"sha256": "5cc7d58328014fe15e3395d201ebbb57c8dbd0d6f51698064aed974f9ead58bd",
|
||||
"bytes": 33146
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/09_multiple_children_schools.yaml",
|
||||
"sha256": "9a0f2647512cfdea680623de4ccd78e3a91520d6471d6905e822f29b05a3e3ea",
|
||||
"bytes": 23765
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/10_travel_rebooking_chain.yaml",
|
||||
"sha256": "e3d1a0907e33311b6123c25a4890449572edebc99b83c691244e5df850488e17",
|
||||
"bytes": 22888
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/11_medical_treatment_evolution.yaml",
|
||||
"sha256": "32703b7e705fbb5721d9db74991b6662951dae7d4a5e99bdc943e1f5183f2b87",
|
||||
"bytes": 23389
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/12_contradictory_financial_instructions.yaml",
|
||||
"sha256": "e2842813485c1684d35371c77c9a439fb8b064febcd15f7210a06856da73b596",
|
||||
"bytes": 21736
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/13_home_services_cascade.yaml",
|
||||
"sha256": "c59d7acce9b2b4571d0128e25ffe7813bd60105ec0aebe8cd6a81f55b99a9f1f",
|
||||
"bytes": 21586
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/14_product_order_modifications.yaml",
|
||||
"sha256": "6d4fb1795b6c2752849f26d82a614aa0358a5bab16dd3c651baa86d3bb6a7017",
|
||||
"bytes": 20856
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/15_employment_negotiation.yaml",
|
||||
"sha256": "d98ce6a5e4fc8758aa011fd98f1016793d89c62f23743b95fb939d1619c71aa8",
|
||||
"bytes": 21116
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/16_family_event_conflicting_input.yaml",
|
||||
"sha256": "e54fe6ccf2de79135778d0cd9391cb2da1ed3ac731b252da9fbdf5b6ab7583a0",
|
||||
"bytes": 22082
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/17_tech_support_cascade.yaml",
|
||||
"sha256": "07c21fd2c800681e6957f9b0d83a23bd931bb60e3d3d5c3303517fd216c6229c",
|
||||
"bytes": 20478
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/18_education_prerequisite_chain.yaml",
|
||||
"sha256": "698dd0d18011559ff242911fec35f97c7d42c851ae02183dbb37ff6c81aec324",
|
||||
"bytes": 20588
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/19_investment_market_response.yaml",
|
||||
"sha256": "1fbfec4ea12194673b8abb3e0f81a0414021536874b5837195a4fec849c9f53b",
|
||||
"bytes": 20608
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/20_healthcare_coverage_changes.yaml",
|
||||
"sha256": "edebb2a0efdf000103e292f305b7aa04600555118fe28a1a61c645585174c76b",
|
||||
"bytes": 22201
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/01_travel_coordination.yaml",
|
||||
"sha256": "6d4ec28d1a5e9ec16d1f8c6367de90827d762b0449fc457c8731b79e29222831",
|
||||
"bytes": 24508
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/02_medical_insurance_coordination.yaml",
|
||||
"sha256": "f4ea8d642ac182431efa9ce70d62985dbfe1ac17e3b980a63aad975fea7b561d",
|
||||
"bytes": 25280
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/03_home_purchase_coordination.yaml",
|
||||
"sha256": "df68870ca4eb344c5a4da6d8f9275ca59fef6edb1ab0a67381e409f656027c33",
|
||||
"bytes": 26717
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/04_warranty_coordination.yaml",
|
||||
"sha256": "08501b6528d39e67e8278720d3b52d26c67e5f03304d82ca17478d85a5183208",
|
||||
"bytes": 27861
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/05_tax_preparation_synthesis.yaml",
|
||||
"sha256": "63a33174166e03d8e747f7ac9b831dcc366b8b70ba7431ac00c6a1004fc569b2",
|
||||
"bytes": 28662
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/06_business_expansion_coordination.yaml",
|
||||
"sha256": "d0b2af52fa1b1e9209d90d6f551ed36d8ca69faf0e58de61f982bc893228baaa",
|
||||
"bytes": 29714
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/07_eldercare_coordination.yaml",
|
||||
"sha256": "71c066861653baac39541b16efd4deae4bad5b275e3260069a0570c8b0df7296",
|
||||
"bytes": 32566
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/08_divorce_settlement_complexity.yaml",
|
||||
"sha256": "1e8ae2d505d5bf50321538bca47e5799b819f749dc932544a94ed812116c0d2f",
|
||||
"bytes": 33565
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/09_vehicle_accident_cascade.yaml",
|
||||
"sha256": "29234d6a933174a9e8b0798e183abe4b60b3edb42bace735c77f6fb115de480e",
|
||||
"bytes": 29687
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/10_education_financing_maze.yaml",
|
||||
"sha256": "7c5fdc9050dfb34c4c3c90d59f9ebfe209e8f3ac93373a34c95a8e1bcc1119fd",
|
||||
"bytes": 29533
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/11_immigration_status_complexity.yaml",
|
||||
"sha256": "15bcd4aeae39f45f354c192b38c77f6071e53579ff6e92e72917bc87dae0a7c4",
|
||||
"bytes": 25066
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/12_real_estate_investment_tangle.yaml",
|
||||
"sha256": "07535fcd40deb5c80d23e5d2faf821b36fcbca42402e01ecd985d5d70545af99",
|
||||
"bytes": 27421
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/13_emergency_medical_cascade.yaml",
|
||||
"sha256": "2a488619f9071ad79691b262db7c2a7087e039897138178a7d9ac7536acc8524",
|
||||
"bytes": 22078
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/14_hidden_medical_insurance_web.yaml",
|
||||
"sha256": "3e8492958ed80ae8aaf85a3ee814fdb5ea6615f4f264073d8fb194add1c6a0b5",
|
||||
"bytes": 24624
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/15_identity_theft_discovery.yaml",
|
||||
"sha256": "27a100b2b13c2fe93f40ac85f0927a0489953b0e4423d78db342b4e48891179a",
|
||||
"bytes": 23071
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/16_cryptocurrency_inheritance_puzzle.yaml",
|
||||
"sha256": "4179a9c39ca7aa1c8a283d2d72dc902d9903b6cc3b68fa0ce3c30b93f506a102",
|
||||
"bytes": 22197
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/17_environmental_contamination_cascade.yaml",
|
||||
"sha256": "556bcaf18251c23f4b0561d7c8d16c5e90e296c3c803e8dfc0f71080a6488904",
|
||||
"bytes": 23054
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/18_genetic_testing_revelation.yaml",
|
||||
"sha256": "a03ec6d38a9005fd566110786ae75583b8fe0382dbd79a82f7ffbfacce0160d5",
|
||||
"bytes": 22923
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/19_employment_fraud_network.yaml",
|
||||
"sha256": "6861efda507da861057e24711255795f389dff0ce2ea66667da2de56ebbb3b10",
|
||||
"bytes": 22985
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/20_medical_malpractice_pattern.yaml",
|
||||
"sha256": "5d986fbcf389d5b8c86844c35184809f77629e53589083663b745ac5a58b467b",
|
||||
"bytes": 23645
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"aggregate": {
|
||||
"advanced_json_cards": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.9,
|
||||
"mean_reward": 0.784375,
|
||||
"hallucination_rate": 0.1
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.6,
|
||||
"mean_reward": 0.7,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.95,
|
||||
"mean_reward": 0.9625,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.8166666666666667,
|
||||
"mean_reward": 0.815625,
|
||||
"hallucination_rate": 0.03333333333333333
|
||||
}
|
||||
}
|
||||
},
|
||||
"token_usage": {
|
||||
"prompt_tokens": 809796,
|
||||
"completion_tokens": 740894,
|
||||
"total_tokens": 1550690
|
||||
},
|
||||
"by_mode": {
|
||||
"notes": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"api_calls": 0,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 0,
|
||||
"mean_per_call": 0
|
||||
}
|
||||
},
|
||||
"advanced_json_cards": {
|
||||
"api_calls": 261,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 809796,
|
||||
"completion_tokens": 740894,
|
||||
"total_tokens": 1550690
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 16943773.900000006,
|
||||
"mean_per_call": 64918.673946360155
|
||||
}
|
||||
}
|
||||
},
|
||||
"api_calls": 261,
|
||||
"errors": 0
|
||||
},
|
||||
"acceptance": {
|
||||
"all_60_cases": true,
|
||||
"twenty_per_layer": true,
|
||||
"all_four_modes": false,
|
||||
"sequential_memory_only": true,
|
||||
"independent_llm_judge": true,
|
||||
"all_calls_succeeded": true,
|
||||
"passed": false
|
||||
}
|
||||
}
|
||||
+493
@@ -0,0 +1,493 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "3-1-and-3-2",
|
||||
"run_id": "20260729T214709Z-3_1_and_3_2-7201b1ff",
|
||||
"created_at": "2026-07-29T21:47:09.564373+00:00",
|
||||
"status": "partial",
|
||||
"run_dir": "/Users/boj/book/ai-agent-book/chapter3/user-memory/validation/runs/20260729T214709Z-3_1_and_3_2-7201b1ff",
|
||||
"artifacts": {
|
||||
"evidence.json": "01d5b16edf10667ea1c5fec41bf6cc9c8878a938f2058a828fb70c1fe39a15c1",
|
||||
"receipts.json": "854340d2d45c8316d2938b80fdf5e8594d9d3f7d2359728edbd3298d00f522ac"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory/run_evaluation.py",
|
||||
"sha256": "dd85135232e65eacb4399acb910bf90ec8ca2d26a5602dc286422dd8cd0d3ace",
|
||||
"bytes": 23589
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/01_bank_account_setup.yaml",
|
||||
"sha256": "e9f788fa393a0f07c83f32e90f05d0b2212355ba62e0a5c3288a469b5feb75b6",
|
||||
"bytes": 11396
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/02_insurance_claim.yaml",
|
||||
"sha256": "ba4d78069e6aaedc1876cbbd3069250ebeafd1def4b619463218ad3a33f80119",
|
||||
"bytes": 13222
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/03_medical_appointment.yaml",
|
||||
"sha256": "42877892ab8720090853ac0fa9050e4097df967a4d88893e25563766e536d3bb",
|
||||
"bytes": 12942
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/04_airline_booking.yaml",
|
||||
"sha256": "76b354c62739d5a23cc71fd94c7a16a0c9552fe8014bc9ddafe262ea0df366de",
|
||||
"bytes": 11836
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/05_internet_service_setup.yaml",
|
||||
"sha256": "f5836b44ae54ee6c220b0261fce73f27e211ed5436b8a52afad17ba6c399658d",
|
||||
"bytes": 12283
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/06_credit_card_application.yaml",
|
||||
"sha256": "7c4358326df8671cad9814a8796df48e5a60cd807ee89a33e6d8523302e8630e",
|
||||
"bytes": 11172
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/07_car_rental.yaml",
|
||||
"sha256": "e497e5819e228f25c58a6f40a29d9d56931eef19c032480ee09eb247a483ebea",
|
||||
"bytes": 10705
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/08_hotel_reservation.yaml",
|
||||
"sha256": "16ddb69e00d880376e869fa454af0b697301c24d7b6b0be20e090ebd9ef64c20",
|
||||
"bytes": 10229
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/09_home_security.yaml",
|
||||
"sha256": "50bc194260c94f8f4e0dd773e5a07dc58d0bf47b72df40aa4a1bc2f7adf5ab3c",
|
||||
"bytes": 11064
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/10_pharmacy_transfer.yaml",
|
||||
"sha256": "d7051d7da1e99b8c662c02cf4bf8a790e72c82efc684562daeb6714f6cd67279",
|
||||
"bytes": 9554
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/11_mortgage_application.yaml",
|
||||
"sha256": "b34269d9b1adc69dc1579317215e5e410c3f85f9f688b5ac283bd419df91a006",
|
||||
"bytes": 14009
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/12_gym_membership.yaml",
|
||||
"sha256": "c530e2215951be9f6553c3ee0c29e68ea596bd9215fa72deda13a44da8ff4d79",
|
||||
"bytes": 12243
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/13_tax_preparation.yaml",
|
||||
"sha256": "ce2af37f2ace66b65cd0f834b8fa727abd7f5399ae260c43c26d07f00bd20e03",
|
||||
"bytes": 14382
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/14_cellphone_upgrade.yaml",
|
||||
"sha256": "26fbf77437deaec9d9bfc635700d3a9f688904d8db594d47b337a2db3de25d83",
|
||||
"bytes": 11402
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/15_college_enrollment.yaml",
|
||||
"sha256": "4252cded12f1e5821d3ad3cbbe270dc4c526cf6ffb86e53bea6db163a3e4f847",
|
||||
"bytes": 12507
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/16_home_renovation.yaml",
|
||||
"sha256": "e66f44d9b3d632cf2afc2759386e8e81dc0edb89b735450fcf5e2e54e2db24fc",
|
||||
"bytes": 11383
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/17_veterinary_care.yaml",
|
||||
"sha256": "712b5726f9d3732d915cb338bf7080d5c048f1156e0718434befa2b86ae92013",
|
||||
"bytes": 10538
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/18_retirement_planning.yaml",
|
||||
"sha256": "46e3876a7d84c70b2bba668f2454991d15066628e83cce97e5fb6abd2ba6558f",
|
||||
"bytes": 10650
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/19_wedding_venue.yaml",
|
||||
"sha256": "b2066bf3897c9b022b63321d9499a254d4b709069b1f5e5d9a0a69dc44026493",
|
||||
"bytes": 10141
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/20_daycare_enrollment.yaml",
|
||||
"sha256": "d6d6a8e6c1130fd693f7aa2febc8e40b49df4848f2ed45ccc849dc78432f315e",
|
||||
"bytes": 11030
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/01_multiple_vehicles.yaml",
|
||||
"sha256": "d00b83e6db7660a2686606218e6b592af883aea201b066a3f6e769d7710e4b48",
|
||||
"bytes": 19514
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/02_multiple_properties.yaml",
|
||||
"sha256": "21b6b80b63eb36481c12317ed63be09d033bcaf7e2526e8abf91d2bdd70d8451",
|
||||
"bytes": 19694
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/03_multiple_credit_cards.yaml",
|
||||
"sha256": "3cde2ffbf8f4f75ed4fc3e1f59c0a1c388582f3223a792e1f496b5ba1b865486",
|
||||
"bytes": 23868
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/04_multiple_subscriptions.yaml",
|
||||
"sha256": "25e2dab7c6f1e12a9ce72646dd702026b19b15d1c8a76aec5d82be4eb0503298",
|
||||
"bytes": 22110
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/05_multiple_bank_accounts.yaml",
|
||||
"sha256": "b30b4c9e503737bd600957e1f3783d046930263ecb78c970056e63b25ee63cc6",
|
||||
"bytes": 21604
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/06_multiple_insurance_policies.yaml",
|
||||
"sha256": "28ec764bd484bb7856941168dc7e557c4eaa791fe54e6b0e466dd6b0ab7776ea",
|
||||
"bytes": 22244
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/07_multiple_medications.yaml",
|
||||
"sha256": "abaa76f5e90ddf3f1a1af1e5bb75fba0b3fae3d77fceb86c54da0d2584c50c24",
|
||||
"bytes": 27025
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/08_multiple_rental_properties.yaml",
|
||||
"sha256": "5cc7d58328014fe15e3395d201ebbb57c8dbd0d6f51698064aed974f9ead58bd",
|
||||
"bytes": 33146
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/09_multiple_children_schools.yaml",
|
||||
"sha256": "9a0f2647512cfdea680623de4ccd78e3a91520d6471d6905e822f29b05a3e3ea",
|
||||
"bytes": 23765
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/10_travel_rebooking_chain.yaml",
|
||||
"sha256": "e3d1a0907e33311b6123c25a4890449572edebc99b83c691244e5df850488e17",
|
||||
"bytes": 22888
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/11_medical_treatment_evolution.yaml",
|
||||
"sha256": "32703b7e705fbb5721d9db74991b6662951dae7d4a5e99bdc943e1f5183f2b87",
|
||||
"bytes": 23389
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/12_contradictory_financial_instructions.yaml",
|
||||
"sha256": "e2842813485c1684d35371c77c9a439fb8b064febcd15f7210a06856da73b596",
|
||||
"bytes": 21736
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/13_home_services_cascade.yaml",
|
||||
"sha256": "c59d7acce9b2b4571d0128e25ffe7813bd60105ec0aebe8cd6a81f55b99a9f1f",
|
||||
"bytes": 21586
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/14_product_order_modifications.yaml",
|
||||
"sha256": "6d4fb1795b6c2752849f26d82a614aa0358a5bab16dd3c651baa86d3bb6a7017",
|
||||
"bytes": 20856
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/15_employment_negotiation.yaml",
|
||||
"sha256": "d98ce6a5e4fc8758aa011fd98f1016793d89c62f23743b95fb939d1619c71aa8",
|
||||
"bytes": 21116
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/16_family_event_conflicting_input.yaml",
|
||||
"sha256": "e54fe6ccf2de79135778d0cd9391cb2da1ed3ac731b252da9fbdf5b6ab7583a0",
|
||||
"bytes": 22082
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/17_tech_support_cascade.yaml",
|
||||
"sha256": "07c21fd2c800681e6957f9b0d83a23bd931bb60e3d3d5c3303517fd216c6229c",
|
||||
"bytes": 20478
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/18_education_prerequisite_chain.yaml",
|
||||
"sha256": "698dd0d18011559ff242911fec35f97c7d42c851ae02183dbb37ff6c81aec324",
|
||||
"bytes": 20588
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/19_investment_market_response.yaml",
|
||||
"sha256": "1fbfec4ea12194673b8abb3e0f81a0414021536874b5837195a4fec849c9f53b",
|
||||
"bytes": 20608
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/20_healthcare_coverage_changes.yaml",
|
||||
"sha256": "edebb2a0efdf000103e292f305b7aa04600555118fe28a1a61c645585174c76b",
|
||||
"bytes": 22201
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/01_travel_coordination.yaml",
|
||||
"sha256": "6d4ec28d1a5e9ec16d1f8c6367de90827d762b0449fc457c8731b79e29222831",
|
||||
"bytes": 24508
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/02_medical_insurance_coordination.yaml",
|
||||
"sha256": "f4ea8d642ac182431efa9ce70d62985dbfe1ac17e3b980a63aad975fea7b561d",
|
||||
"bytes": 25280
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/03_home_purchase_coordination.yaml",
|
||||
"sha256": "df68870ca4eb344c5a4da6d8f9275ca59fef6edb1ab0a67381e409f656027c33",
|
||||
"bytes": 26717
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/04_warranty_coordination.yaml",
|
||||
"sha256": "08501b6528d39e67e8278720d3b52d26c67e5f03304d82ca17478d85a5183208",
|
||||
"bytes": 27861
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/05_tax_preparation_synthesis.yaml",
|
||||
"sha256": "63a33174166e03d8e747f7ac9b831dcc366b8b70ba7431ac00c6a1004fc569b2",
|
||||
"bytes": 28662
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/06_business_expansion_coordination.yaml",
|
||||
"sha256": "d0b2af52fa1b1e9209d90d6f551ed36d8ca69faf0e58de61f982bc893228baaa",
|
||||
"bytes": 29714
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/07_eldercare_coordination.yaml",
|
||||
"sha256": "71c066861653baac39541b16efd4deae4bad5b275e3260069a0570c8b0df7296",
|
||||
"bytes": 32566
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/08_divorce_settlement_complexity.yaml",
|
||||
"sha256": "1e8ae2d505d5bf50321538bca47e5799b819f749dc932544a94ed812116c0d2f",
|
||||
"bytes": 33565
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/09_vehicle_accident_cascade.yaml",
|
||||
"sha256": "29234d6a933174a9e8b0798e183abe4b60b3edb42bace735c77f6fb115de480e",
|
||||
"bytes": 29687
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/10_education_financing_maze.yaml",
|
||||
"sha256": "7c5fdc9050dfb34c4c3c90d59f9ebfe209e8f3ac93373a34c95a8e1bcc1119fd",
|
||||
"bytes": 29533
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/11_immigration_status_complexity.yaml",
|
||||
"sha256": "15bcd4aeae39f45f354c192b38c77f6071e53579ff6e92e72917bc87dae0a7c4",
|
||||
"bytes": 25066
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/12_real_estate_investment_tangle.yaml",
|
||||
"sha256": "07535fcd40deb5c80d23e5d2faf821b36fcbca42402e01ecd985d5d70545af99",
|
||||
"bytes": 27421
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/13_emergency_medical_cascade.yaml",
|
||||
"sha256": "2a488619f9071ad79691b262db7c2a7087e039897138178a7d9ac7536acc8524",
|
||||
"bytes": 22078
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/14_hidden_medical_insurance_web.yaml",
|
||||
"sha256": "3e8492958ed80ae8aaf85a3ee814fdb5ea6615f4f264073d8fb194add1c6a0b5",
|
||||
"bytes": 24624
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/15_identity_theft_discovery.yaml",
|
||||
"sha256": "27a100b2b13c2fe93f40ac85f0927a0489953b0e4423d78db342b4e48891179a",
|
||||
"bytes": 23071
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/16_cryptocurrency_inheritance_puzzle.yaml",
|
||||
"sha256": "4179a9c39ca7aa1c8a283d2d72dc902d9903b6cc3b68fa0ce3c30b93f506a102",
|
||||
"bytes": 22197
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/17_environmental_contamination_cascade.yaml",
|
||||
"sha256": "556bcaf18251c23f4b0561d7c8d16c5e90e296c3c803e8dfc0f71080a6488904",
|
||||
"bytes": 23054
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/18_genetic_testing_revelation.yaml",
|
||||
"sha256": "a03ec6d38a9005fd566110786ae75583b8fe0382dbd79a82f7ffbfacce0160d5",
|
||||
"bytes": 22923
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/19_employment_fraud_network.yaml",
|
||||
"sha256": "6861efda507da861057e24711255795f389dff0ce2ea66667da2de56ebbb3b10",
|
||||
"bytes": 22985
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/20_medical_malpractice_pattern.yaml",
|
||||
"sha256": "5d986fbcf389d5b8c86844c35184809f77629e53589083663b745ac5a58b467b",
|
||||
"bytes": 23645
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"aggregate": {
|
||||
"advanced_json_cards": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.9,
|
||||
"mean_reward": 0.784375,
|
||||
"hallucination_rate": 0.1
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.6,
|
||||
"mean_reward": 0.7,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.95,
|
||||
"mean_reward": 0.9625,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.8166666666666667,
|
||||
"mean_reward": 0.815625,
|
||||
"hallucination_rate": 0.03333333333333333
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.85,
|
||||
"mean_reward": 0.759375,
|
||||
"hallucination_rate": 0.1
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.8,
|
||||
"mean_reward": 0.85,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.95,
|
||||
"mean_reward": 0.9625,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.8666666666666667,
|
||||
"mean_reward": 0.8572916666666667,
|
||||
"hallucination_rate": 0.03333333333333333
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.95,
|
||||
"mean_reward": 0.834375,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.7,
|
||||
"mean_reward": 0.753125,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer3": {
|
||||
"n": 19,
|
||||
"pass_rate": 0.8947368421052632,
|
||||
"mean_reward": 0.8947368421052632,
|
||||
"hallucination_rate": 0.10526315789473684
|
||||
},
|
||||
"overall": {
|
||||
"n": 59,
|
||||
"pass_rate": 0.847457627118644,
|
||||
"mean_reward": 0.826271186440678,
|
||||
"hallucination_rate": 0.06779661016949153
|
||||
}
|
||||
},
|
||||
"notes": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.9,
|
||||
"mean_reward": 0.8,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.65,
|
||||
"mean_reward": 0.721875,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.85,
|
||||
"mean_reward": 0.8875,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.8,
|
||||
"mean_reward": 0.803125,
|
||||
"hallucination_rate": 0.03333333333333333
|
||||
}
|
||||
}
|
||||
},
|
||||
"token_usage": {
|
||||
"prompt_tokens": 2751195,
|
||||
"completion_tokens": 1985040,
|
||||
"total_tokens": 4736235
|
||||
},
|
||||
"by_mode": {
|
||||
"notes": {
|
||||
"api_calls": 260,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 662861,
|
||||
"completion_tokens": 438410,
|
||||
"total_tokens": 1101271
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 9920610.951999996,
|
||||
"mean_per_call": 38156.19596923077
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"api_calls": 260,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 635218,
|
||||
"completion_tokens": 352250,
|
||||
"total_tokens": 987468
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 8365401.234999998,
|
||||
"mean_per_call": 32174.620134615383
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"api_calls": 255,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 643320,
|
||||
"completion_tokens": 453486,
|
||||
"total_tokens": 1096806
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 10259127.337999992,
|
||||
"mean_per_call": 40231.87191372549
|
||||
}
|
||||
},
|
||||
"advanced_json_cards": {
|
||||
"api_calls": 261,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 809796,
|
||||
"completion_tokens": 740894,
|
||||
"total_tokens": 1550690
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 16943773.900000006,
|
||||
"mean_per_call": 64918.673946360155
|
||||
}
|
||||
}
|
||||
},
|
||||
"api_calls": 1036,
|
||||
"errors": 1
|
||||
},
|
||||
"acceptance": {
|
||||
"all_60_cases": true,
|
||||
"twenty_per_layer": true,
|
||||
"all_four_modes": true,
|
||||
"sequential_memory_only": true,
|
||||
"independent_llm_judge": true,
|
||||
"all_calls_succeeded": false,
|
||||
"passed": false
|
||||
}
|
||||
}
|
||||
+493
@@ -0,0 +1,493 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "3-1-and-3-2",
|
||||
"run_id": "20260729T214915Z-3_1_and_3_2-95c7b98f",
|
||||
"created_at": "2026-07-29T21:49:15.412512+00:00",
|
||||
"status": "passed",
|
||||
"run_dir": "/Users/boj/book/ai-agent-book/chapter3/user-memory/validation/runs/20260729T214915Z-3_1_and_3_2-95c7b98f",
|
||||
"artifacts": {
|
||||
"evidence.json": "eda4172bf58d8ecc0607813fabde50db082a91c53f9388564d45118f361fc769",
|
||||
"receipts.json": "eb739d9a82c937608b7e4edb746cbda6cefe44982e80bba8dd63e935de05d174"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory/run_evaluation.py",
|
||||
"sha256": "dd85135232e65eacb4399acb910bf90ec8ca2d26a5602dc286422dd8cd0d3ace",
|
||||
"bytes": 23589
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/01_bank_account_setup.yaml",
|
||||
"sha256": "e9f788fa393a0f07c83f32e90f05d0b2212355ba62e0a5c3288a469b5feb75b6",
|
||||
"bytes": 11396
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/02_insurance_claim.yaml",
|
||||
"sha256": "ba4d78069e6aaedc1876cbbd3069250ebeafd1def4b619463218ad3a33f80119",
|
||||
"bytes": 13222
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/03_medical_appointment.yaml",
|
||||
"sha256": "42877892ab8720090853ac0fa9050e4097df967a4d88893e25563766e536d3bb",
|
||||
"bytes": 12942
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/04_airline_booking.yaml",
|
||||
"sha256": "76b354c62739d5a23cc71fd94c7a16a0c9552fe8014bc9ddafe262ea0df366de",
|
||||
"bytes": 11836
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/05_internet_service_setup.yaml",
|
||||
"sha256": "f5836b44ae54ee6c220b0261fce73f27e211ed5436b8a52afad17ba6c399658d",
|
||||
"bytes": 12283
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/06_credit_card_application.yaml",
|
||||
"sha256": "7c4358326df8671cad9814a8796df48e5a60cd807ee89a33e6d8523302e8630e",
|
||||
"bytes": 11172
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/07_car_rental.yaml",
|
||||
"sha256": "e497e5819e228f25c58a6f40a29d9d56931eef19c032480ee09eb247a483ebea",
|
||||
"bytes": 10705
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/08_hotel_reservation.yaml",
|
||||
"sha256": "16ddb69e00d880376e869fa454af0b697301c24d7b6b0be20e090ebd9ef64c20",
|
||||
"bytes": 10229
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/09_home_security.yaml",
|
||||
"sha256": "50bc194260c94f8f4e0dd773e5a07dc58d0bf47b72df40aa4a1bc2f7adf5ab3c",
|
||||
"bytes": 11064
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/10_pharmacy_transfer.yaml",
|
||||
"sha256": "d7051d7da1e99b8c662c02cf4bf8a790e72c82efc684562daeb6714f6cd67279",
|
||||
"bytes": 9554
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/11_mortgage_application.yaml",
|
||||
"sha256": "b34269d9b1adc69dc1579317215e5e410c3f85f9f688b5ac283bd419df91a006",
|
||||
"bytes": 14009
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/12_gym_membership.yaml",
|
||||
"sha256": "c530e2215951be9f6553c3ee0c29e68ea596bd9215fa72deda13a44da8ff4d79",
|
||||
"bytes": 12243
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/13_tax_preparation.yaml",
|
||||
"sha256": "ce2af37f2ace66b65cd0f834b8fa727abd7f5399ae260c43c26d07f00bd20e03",
|
||||
"bytes": 14382
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/14_cellphone_upgrade.yaml",
|
||||
"sha256": "26fbf77437deaec9d9bfc635700d3a9f688904d8db594d47b337a2db3de25d83",
|
||||
"bytes": 11402
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/15_college_enrollment.yaml",
|
||||
"sha256": "4252cded12f1e5821d3ad3cbbe270dc4c526cf6ffb86e53bea6db163a3e4f847",
|
||||
"bytes": 12507
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/16_home_renovation.yaml",
|
||||
"sha256": "e66f44d9b3d632cf2afc2759386e8e81dc0edb89b735450fcf5e2e54e2db24fc",
|
||||
"bytes": 11383
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/17_veterinary_care.yaml",
|
||||
"sha256": "712b5726f9d3732d915cb338bf7080d5c048f1156e0718434befa2b86ae92013",
|
||||
"bytes": 10538
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/18_retirement_planning.yaml",
|
||||
"sha256": "46e3876a7d84c70b2bba668f2454991d15066628e83cce97e5fb6abd2ba6558f",
|
||||
"bytes": 10650
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/19_wedding_venue.yaml",
|
||||
"sha256": "b2066bf3897c9b022b63321d9499a254d4b709069b1f5e5d9a0a69dc44026493",
|
||||
"bytes": 10141
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/20_daycare_enrollment.yaml",
|
||||
"sha256": "d6d6a8e6c1130fd693f7aa2febc8e40b49df4848f2ed45ccc849dc78432f315e",
|
||||
"bytes": 11030
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/01_multiple_vehicles.yaml",
|
||||
"sha256": "d00b83e6db7660a2686606218e6b592af883aea201b066a3f6e769d7710e4b48",
|
||||
"bytes": 19514
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/02_multiple_properties.yaml",
|
||||
"sha256": "21b6b80b63eb36481c12317ed63be09d033bcaf7e2526e8abf91d2bdd70d8451",
|
||||
"bytes": 19694
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/03_multiple_credit_cards.yaml",
|
||||
"sha256": "3cde2ffbf8f4f75ed4fc3e1f59c0a1c388582f3223a792e1f496b5ba1b865486",
|
||||
"bytes": 23868
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/04_multiple_subscriptions.yaml",
|
||||
"sha256": "25e2dab7c6f1e12a9ce72646dd702026b19b15d1c8a76aec5d82be4eb0503298",
|
||||
"bytes": 22110
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/05_multiple_bank_accounts.yaml",
|
||||
"sha256": "b30b4c9e503737bd600957e1f3783d046930263ecb78c970056e63b25ee63cc6",
|
||||
"bytes": 21604
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/06_multiple_insurance_policies.yaml",
|
||||
"sha256": "28ec764bd484bb7856941168dc7e557c4eaa791fe54e6b0e466dd6b0ab7776ea",
|
||||
"bytes": 22244
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/07_multiple_medications.yaml",
|
||||
"sha256": "abaa76f5e90ddf3f1a1af1e5bb75fba0b3fae3d77fceb86c54da0d2584c50c24",
|
||||
"bytes": 27025
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/08_multiple_rental_properties.yaml",
|
||||
"sha256": "5cc7d58328014fe15e3395d201ebbb57c8dbd0d6f51698064aed974f9ead58bd",
|
||||
"bytes": 33146
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/09_multiple_children_schools.yaml",
|
||||
"sha256": "9a0f2647512cfdea680623de4ccd78e3a91520d6471d6905e822f29b05a3e3ea",
|
||||
"bytes": 23765
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/10_travel_rebooking_chain.yaml",
|
||||
"sha256": "e3d1a0907e33311b6123c25a4890449572edebc99b83c691244e5df850488e17",
|
||||
"bytes": 22888
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/11_medical_treatment_evolution.yaml",
|
||||
"sha256": "32703b7e705fbb5721d9db74991b6662951dae7d4a5e99bdc943e1f5183f2b87",
|
||||
"bytes": 23389
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/12_contradictory_financial_instructions.yaml",
|
||||
"sha256": "e2842813485c1684d35371c77c9a439fb8b064febcd15f7210a06856da73b596",
|
||||
"bytes": 21736
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/13_home_services_cascade.yaml",
|
||||
"sha256": "c59d7acce9b2b4571d0128e25ffe7813bd60105ec0aebe8cd6a81f55b99a9f1f",
|
||||
"bytes": 21586
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/14_product_order_modifications.yaml",
|
||||
"sha256": "6d4fb1795b6c2752849f26d82a614aa0358a5bab16dd3c651baa86d3bb6a7017",
|
||||
"bytes": 20856
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/15_employment_negotiation.yaml",
|
||||
"sha256": "d98ce6a5e4fc8758aa011fd98f1016793d89c62f23743b95fb939d1619c71aa8",
|
||||
"bytes": 21116
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/16_family_event_conflicting_input.yaml",
|
||||
"sha256": "e54fe6ccf2de79135778d0cd9391cb2da1ed3ac731b252da9fbdf5b6ab7583a0",
|
||||
"bytes": 22082
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/17_tech_support_cascade.yaml",
|
||||
"sha256": "07c21fd2c800681e6957f9b0d83a23bd931bb60e3d3d5c3303517fd216c6229c",
|
||||
"bytes": 20478
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/18_education_prerequisite_chain.yaml",
|
||||
"sha256": "698dd0d18011559ff242911fec35f97c7d42c851ae02183dbb37ff6c81aec324",
|
||||
"bytes": 20588
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/19_investment_market_response.yaml",
|
||||
"sha256": "1fbfec4ea12194673b8abb3e0f81a0414021536874b5837195a4fec849c9f53b",
|
||||
"bytes": 20608
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/20_healthcare_coverage_changes.yaml",
|
||||
"sha256": "edebb2a0efdf000103e292f305b7aa04600555118fe28a1a61c645585174c76b",
|
||||
"bytes": 22201
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/01_travel_coordination.yaml",
|
||||
"sha256": "6d4ec28d1a5e9ec16d1f8c6367de90827d762b0449fc457c8731b79e29222831",
|
||||
"bytes": 24508
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/02_medical_insurance_coordination.yaml",
|
||||
"sha256": "f4ea8d642ac182431efa9ce70d62985dbfe1ac17e3b980a63aad975fea7b561d",
|
||||
"bytes": 25280
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/03_home_purchase_coordination.yaml",
|
||||
"sha256": "df68870ca4eb344c5a4da6d8f9275ca59fef6edb1ab0a67381e409f656027c33",
|
||||
"bytes": 26717
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/04_warranty_coordination.yaml",
|
||||
"sha256": "08501b6528d39e67e8278720d3b52d26c67e5f03304d82ca17478d85a5183208",
|
||||
"bytes": 27861
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/05_tax_preparation_synthesis.yaml",
|
||||
"sha256": "63a33174166e03d8e747f7ac9b831dcc366b8b70ba7431ac00c6a1004fc569b2",
|
||||
"bytes": 28662
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/06_business_expansion_coordination.yaml",
|
||||
"sha256": "d0b2af52fa1b1e9209d90d6f551ed36d8ca69faf0e58de61f982bc893228baaa",
|
||||
"bytes": 29714
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/07_eldercare_coordination.yaml",
|
||||
"sha256": "71c066861653baac39541b16efd4deae4bad5b275e3260069a0570c8b0df7296",
|
||||
"bytes": 32566
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/08_divorce_settlement_complexity.yaml",
|
||||
"sha256": "1e8ae2d505d5bf50321538bca47e5799b819f749dc932544a94ed812116c0d2f",
|
||||
"bytes": 33565
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/09_vehicle_accident_cascade.yaml",
|
||||
"sha256": "29234d6a933174a9e8b0798e183abe4b60b3edb42bace735c77f6fb115de480e",
|
||||
"bytes": 29687
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/10_education_financing_maze.yaml",
|
||||
"sha256": "7c5fdc9050dfb34c4c3c90d59f9ebfe209e8f3ac93373a34c95a8e1bcc1119fd",
|
||||
"bytes": 29533
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/11_immigration_status_complexity.yaml",
|
||||
"sha256": "15bcd4aeae39f45f354c192b38c77f6071e53579ff6e92e72917bc87dae0a7c4",
|
||||
"bytes": 25066
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/12_real_estate_investment_tangle.yaml",
|
||||
"sha256": "07535fcd40deb5c80d23e5d2faf821b36fcbca42402e01ecd985d5d70545af99",
|
||||
"bytes": 27421
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/13_emergency_medical_cascade.yaml",
|
||||
"sha256": "2a488619f9071ad79691b262db7c2a7087e039897138178a7d9ac7536acc8524",
|
||||
"bytes": 22078
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/14_hidden_medical_insurance_web.yaml",
|
||||
"sha256": "3e8492958ed80ae8aaf85a3ee814fdb5ea6615f4f264073d8fb194add1c6a0b5",
|
||||
"bytes": 24624
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/15_identity_theft_discovery.yaml",
|
||||
"sha256": "27a100b2b13c2fe93f40ac85f0927a0489953b0e4423d78db342b4e48891179a",
|
||||
"bytes": 23071
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/16_cryptocurrency_inheritance_puzzle.yaml",
|
||||
"sha256": "4179a9c39ca7aa1c8a283d2d72dc902d9903b6cc3b68fa0ce3c30b93f506a102",
|
||||
"bytes": 22197
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/17_environmental_contamination_cascade.yaml",
|
||||
"sha256": "556bcaf18251c23f4b0561d7c8d16c5e90e296c3c803e8dfc0f71080a6488904",
|
||||
"bytes": 23054
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/18_genetic_testing_revelation.yaml",
|
||||
"sha256": "a03ec6d38a9005fd566110786ae75583b8fe0382dbd79a82f7ffbfacce0160d5",
|
||||
"bytes": 22923
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/19_employment_fraud_network.yaml",
|
||||
"sha256": "6861efda507da861057e24711255795f389dff0ce2ea66667da2de56ebbb3b10",
|
||||
"bytes": 22985
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/20_medical_malpractice_pattern.yaml",
|
||||
"sha256": "5d986fbcf389d5b8c86844c35184809f77629e53589083663b745ac5a58b467b",
|
||||
"bytes": 23645
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"aggregate": {
|
||||
"advanced_json_cards": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.9,
|
||||
"mean_reward": 0.784375,
|
||||
"hallucination_rate": 0.1
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.6,
|
||||
"mean_reward": 0.7,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.95,
|
||||
"mean_reward": 0.9625,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.8166666666666667,
|
||||
"mean_reward": 0.815625,
|
||||
"hallucination_rate": 0.03333333333333333
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.85,
|
||||
"mean_reward": 0.759375,
|
||||
"hallucination_rate": 0.1
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.8,
|
||||
"mean_reward": 0.85,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.95,
|
||||
"mean_reward": 0.9625,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.8666666666666667,
|
||||
"mean_reward": 0.8572916666666667,
|
||||
"hallucination_rate": 0.03333333333333333
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.95,
|
||||
"mean_reward": 0.834375,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.7,
|
||||
"mean_reward": 0.753125,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.9,
|
||||
"mean_reward": 0.9,
|
||||
"hallucination_rate": 0.1
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.85,
|
||||
"mean_reward": 0.8291666666666667,
|
||||
"hallucination_rate": 0.06666666666666667
|
||||
}
|
||||
},
|
||||
"notes": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.9,
|
||||
"mean_reward": 0.8,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.65,
|
||||
"mean_reward": 0.721875,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.85,
|
||||
"mean_reward": 0.8875,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.8,
|
||||
"mean_reward": 0.803125,
|
||||
"hallucination_rate": 0.03333333333333333
|
||||
}
|
||||
}
|
||||
},
|
||||
"token_usage": {
|
||||
"prompt_tokens": 2764882,
|
||||
"completion_tokens": 2000654,
|
||||
"total_tokens": 4765536
|
||||
},
|
||||
"by_mode": {
|
||||
"notes": {
|
||||
"api_calls": 260,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 662861,
|
||||
"completion_tokens": 438410,
|
||||
"total_tokens": 1101271
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 9920610.951999996,
|
||||
"mean_per_call": 38156.19596923077
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"api_calls": 260,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 635218,
|
||||
"completion_tokens": 352250,
|
||||
"total_tokens": 987468
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 8365401.234999998,
|
||||
"mean_per_call": 32174.620134615383
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"api_calls": 261,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 657007,
|
||||
"completion_tokens": 469100,
|
||||
"total_tokens": 1126107
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 10583750.72799999,
|
||||
"mean_per_call": 40550.76907279694
|
||||
}
|
||||
},
|
||||
"advanced_json_cards": {
|
||||
"api_calls": 261,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 809796,
|
||||
"completion_tokens": 740894,
|
||||
"total_tokens": 1550690
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 16943773.900000006,
|
||||
"mean_per_call": 64918.673946360155
|
||||
}
|
||||
}
|
||||
},
|
||||
"api_calls": 1042,
|
||||
"errors": 0
|
||||
},
|
||||
"acceptance": {
|
||||
"all_60_cases": true,
|
||||
"twenty_per_layer": true,
|
||||
"all_four_modes": true,
|
||||
"sequential_memory_only": true,
|
||||
"independent_llm_judge": true,
|
||||
"all_calls_succeeded": true,
|
||||
"passed": true
|
||||
}
|
||||
}
|
||||
+493
@@ -0,0 +1,493 @@
|
||||
{
|
||||
"schema_version": "chapter3-evidence-v1",
|
||||
"experiment": "3-1-and-3-2",
|
||||
"run_id": "20260730T043652Z-3_1_and_3_2-16200e94",
|
||||
"created_at": "2026-07-30T04:36:53.131949+00:00",
|
||||
"status": "passed",
|
||||
"run_dir": "/Users/boj/book/ai-agent-book/chapter3/user-memory/validation/runs/20260730T043652Z-3_1_and_3_2-16200e94",
|
||||
"artifacts": {
|
||||
"evidence.json": "b29968a6780727a56ce641481f1be7f454865ab9017fdb184f0cf60ad00de724",
|
||||
"receipts.json": "89bd28e4a9714e804c0b46614ae61f4ae9b52a16dcb5eea297dc30e559f61dae"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory/run_evaluation.py",
|
||||
"sha256": "a69f0039de306cbda2ef8920195a9720b7435e9a072b7c7f1823c5e3d61dc286",
|
||||
"bytes": 23587
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/01_bank_account_setup.yaml",
|
||||
"sha256": "e9f788fa393a0f07c83f32e90f05d0b2212355ba62e0a5c3288a469b5feb75b6",
|
||||
"bytes": 11396
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/02_insurance_claim.yaml",
|
||||
"sha256": "ba4d78069e6aaedc1876cbbd3069250ebeafd1def4b619463218ad3a33f80119",
|
||||
"bytes": 13222
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/03_medical_appointment.yaml",
|
||||
"sha256": "42877892ab8720090853ac0fa9050e4097df967a4d88893e25563766e536d3bb",
|
||||
"bytes": 12942
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/04_airline_booking.yaml",
|
||||
"sha256": "76b354c62739d5a23cc71fd94c7a16a0c9552fe8014bc9ddafe262ea0df366de",
|
||||
"bytes": 11836
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/05_internet_service_setup.yaml",
|
||||
"sha256": "f5836b44ae54ee6c220b0261fce73f27e211ed5436b8a52afad17ba6c399658d",
|
||||
"bytes": 12283
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/06_credit_card_application.yaml",
|
||||
"sha256": "7c4358326df8671cad9814a8796df48e5a60cd807ee89a33e6d8523302e8630e",
|
||||
"bytes": 11172
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/07_car_rental.yaml",
|
||||
"sha256": "e497e5819e228f25c58a6f40a29d9d56931eef19c032480ee09eb247a483ebea",
|
||||
"bytes": 10705
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/08_hotel_reservation.yaml",
|
||||
"sha256": "16ddb69e00d880376e869fa454af0b697301c24d7b6b0be20e090ebd9ef64c20",
|
||||
"bytes": 10229
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/09_home_security.yaml",
|
||||
"sha256": "50bc194260c94f8f4e0dd773e5a07dc58d0bf47b72df40aa4a1bc2f7adf5ab3c",
|
||||
"bytes": 11064
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/10_pharmacy_transfer.yaml",
|
||||
"sha256": "d7051d7da1e99b8c662c02cf4bf8a790e72c82efc684562daeb6714f6cd67279",
|
||||
"bytes": 9554
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/11_mortgage_application.yaml",
|
||||
"sha256": "b34269d9b1adc69dc1579317215e5e410c3f85f9f688b5ac283bd419df91a006",
|
||||
"bytes": 14009
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/12_gym_membership.yaml",
|
||||
"sha256": "c530e2215951be9f6553c3ee0c29e68ea596bd9215fa72deda13a44da8ff4d79",
|
||||
"bytes": 12243
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/13_tax_preparation.yaml",
|
||||
"sha256": "ce2af37f2ace66b65cd0f834b8fa727abd7f5399ae260c43c26d07f00bd20e03",
|
||||
"bytes": 14382
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/14_cellphone_upgrade.yaml",
|
||||
"sha256": "26fbf77437deaec9d9bfc635700d3a9f688904d8db594d47b337a2db3de25d83",
|
||||
"bytes": 11402
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/15_college_enrollment.yaml",
|
||||
"sha256": "4252cded12f1e5821d3ad3cbbe270dc4c526cf6ffb86e53bea6db163a3e4f847",
|
||||
"bytes": 12507
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/16_home_renovation.yaml",
|
||||
"sha256": "e66f44d9b3d632cf2afc2759386e8e81dc0edb89b735450fcf5e2e54e2db24fc",
|
||||
"bytes": 11383
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/17_veterinary_care.yaml",
|
||||
"sha256": "712b5726f9d3732d915cb338bf7080d5c048f1156e0718434befa2b86ae92013",
|
||||
"bytes": 10538
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/18_retirement_planning.yaml",
|
||||
"sha256": "46e3876a7d84c70b2bba668f2454991d15066628e83cce97e5fb6abd2ba6558f",
|
||||
"bytes": 10650
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/19_wedding_venue.yaml",
|
||||
"sha256": "b2066bf3897c9b022b63321d9499a254d4b709069b1f5e5d9a0a69dc44026493",
|
||||
"bytes": 10141
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/20_daycare_enrollment.yaml",
|
||||
"sha256": "d6d6a8e6c1130fd693f7aa2febc8e40b49df4848f2ed45ccc849dc78432f315e",
|
||||
"bytes": 11030
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/01_multiple_vehicles.yaml",
|
||||
"sha256": "d00b83e6db7660a2686606218e6b592af883aea201b066a3f6e769d7710e4b48",
|
||||
"bytes": 19514
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/02_multiple_properties.yaml",
|
||||
"sha256": "21b6b80b63eb36481c12317ed63be09d033bcaf7e2526e8abf91d2bdd70d8451",
|
||||
"bytes": 19694
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/03_multiple_credit_cards.yaml",
|
||||
"sha256": "3cde2ffbf8f4f75ed4fc3e1f59c0a1c388582f3223a792e1f496b5ba1b865486",
|
||||
"bytes": 23868
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/04_multiple_subscriptions.yaml",
|
||||
"sha256": "25e2dab7c6f1e12a9ce72646dd702026b19b15d1c8a76aec5d82be4eb0503298",
|
||||
"bytes": 22110
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/05_multiple_bank_accounts.yaml",
|
||||
"sha256": "b30b4c9e503737bd600957e1f3783d046930263ecb78c970056e63b25ee63cc6",
|
||||
"bytes": 21604
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/06_multiple_insurance_policies.yaml",
|
||||
"sha256": "28ec764bd484bb7856941168dc7e557c4eaa791fe54e6b0e466dd6b0ab7776ea",
|
||||
"bytes": 22244
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/07_multiple_medications.yaml",
|
||||
"sha256": "abaa76f5e90ddf3f1a1af1e5bb75fba0b3fae3d77fceb86c54da0d2584c50c24",
|
||||
"bytes": 27025
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/08_multiple_rental_properties.yaml",
|
||||
"sha256": "5cc7d58328014fe15e3395d201ebbb57c8dbd0d6f51698064aed974f9ead58bd",
|
||||
"bytes": 33146
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/09_multiple_children_schools.yaml",
|
||||
"sha256": "9a0f2647512cfdea680623de4ccd78e3a91520d6471d6905e822f29b05a3e3ea",
|
||||
"bytes": 23765
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/10_travel_rebooking_chain.yaml",
|
||||
"sha256": "e3d1a0907e33311b6123c25a4890449572edebc99b83c691244e5df850488e17",
|
||||
"bytes": 22888
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/11_medical_treatment_evolution.yaml",
|
||||
"sha256": "32703b7e705fbb5721d9db74991b6662951dae7d4a5e99bdc943e1f5183f2b87",
|
||||
"bytes": 23389
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/12_contradictory_financial_instructions.yaml",
|
||||
"sha256": "e2842813485c1684d35371c77c9a439fb8b064febcd15f7210a06856da73b596",
|
||||
"bytes": 21736
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/13_home_services_cascade.yaml",
|
||||
"sha256": "c59d7acce9b2b4571d0128e25ffe7813bd60105ec0aebe8cd6a81f55b99a9f1f",
|
||||
"bytes": 21586
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/14_product_order_modifications.yaml",
|
||||
"sha256": "6d4fb1795b6c2752849f26d82a614aa0358a5bab16dd3c651baa86d3bb6a7017",
|
||||
"bytes": 20856
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/15_employment_negotiation.yaml",
|
||||
"sha256": "d98ce6a5e4fc8758aa011fd98f1016793d89c62f23743b95fb939d1619c71aa8",
|
||||
"bytes": 21116
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/16_family_event_conflicting_input.yaml",
|
||||
"sha256": "e54fe6ccf2de79135778d0cd9391cb2da1ed3ac731b252da9fbdf5b6ab7583a0",
|
||||
"bytes": 22082
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/17_tech_support_cascade.yaml",
|
||||
"sha256": "07c21fd2c800681e6957f9b0d83a23bd931bb60e3d3d5c3303517fd216c6229c",
|
||||
"bytes": 20478
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/18_education_prerequisite_chain.yaml",
|
||||
"sha256": "698dd0d18011559ff242911fec35f97c7d42c851ae02183dbb37ff6c81aec324",
|
||||
"bytes": 20588
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/19_investment_market_response.yaml",
|
||||
"sha256": "1fbfec4ea12194673b8abb3e0f81a0414021536874b5837195a4fec849c9f53b",
|
||||
"bytes": 20608
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/20_healthcare_coverage_changes.yaml",
|
||||
"sha256": "edebb2a0efdf000103e292f305b7aa04600555118fe28a1a61c645585174c76b",
|
||||
"bytes": 22201
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/01_travel_coordination.yaml",
|
||||
"sha256": "6d4ec28d1a5e9ec16d1f8c6367de90827d762b0449fc457c8731b79e29222831",
|
||||
"bytes": 24508
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/02_medical_insurance_coordination.yaml",
|
||||
"sha256": "f4ea8d642ac182431efa9ce70d62985dbfe1ac17e3b980a63aad975fea7b561d",
|
||||
"bytes": 25280
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/03_home_purchase_coordination.yaml",
|
||||
"sha256": "df68870ca4eb344c5a4da6d8f9275ca59fef6edb1ab0a67381e409f656027c33",
|
||||
"bytes": 26717
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/04_warranty_coordination.yaml",
|
||||
"sha256": "08501b6528d39e67e8278720d3b52d26c67e5f03304d82ca17478d85a5183208",
|
||||
"bytes": 27861
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/05_tax_preparation_synthesis.yaml",
|
||||
"sha256": "63a33174166e03d8e747f7ac9b831dcc366b8b70ba7431ac00c6a1004fc569b2",
|
||||
"bytes": 28662
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/06_business_expansion_coordination.yaml",
|
||||
"sha256": "d0b2af52fa1b1e9209d90d6f551ed36d8ca69faf0e58de61f982bc893228baaa",
|
||||
"bytes": 29714
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/07_eldercare_coordination.yaml",
|
||||
"sha256": "71c066861653baac39541b16efd4deae4bad5b275e3260069a0570c8b0df7296",
|
||||
"bytes": 32566
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/08_divorce_settlement_complexity.yaml",
|
||||
"sha256": "1e8ae2d505d5bf50321538bca47e5799b819f749dc932544a94ed812116c0d2f",
|
||||
"bytes": 33565
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/09_vehicle_accident_cascade.yaml",
|
||||
"sha256": "29234d6a933174a9e8b0798e183abe4b60b3edb42bace735c77f6fb115de480e",
|
||||
"bytes": 29687
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/10_education_financing_maze.yaml",
|
||||
"sha256": "7c5fdc9050dfb34c4c3c90d59f9ebfe209e8f3ac93373a34c95a8e1bcc1119fd",
|
||||
"bytes": 29533
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/11_immigration_status_complexity.yaml",
|
||||
"sha256": "15bcd4aeae39f45f354c192b38c77f6071e53579ff6e92e72917bc87dae0a7c4",
|
||||
"bytes": 25066
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/12_real_estate_investment_tangle.yaml",
|
||||
"sha256": "07535fcd40deb5c80d23e5d2faf821b36fcbca42402e01ecd985d5d70545af99",
|
||||
"bytes": 27421
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/13_emergency_medical_cascade.yaml",
|
||||
"sha256": "2a488619f9071ad79691b262db7c2a7087e039897138178a7d9ac7536acc8524",
|
||||
"bytes": 22078
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/14_hidden_medical_insurance_web.yaml",
|
||||
"sha256": "3e8492958ed80ae8aaf85a3ee814fdb5ea6615f4f264073d8fb194add1c6a0b5",
|
||||
"bytes": 24624
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/15_identity_theft_discovery.yaml",
|
||||
"sha256": "27a100b2b13c2fe93f40ac85f0927a0489953b0e4423d78db342b4e48891179a",
|
||||
"bytes": 23071
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/16_cryptocurrency_inheritance_puzzle.yaml",
|
||||
"sha256": "4179a9c39ca7aa1c8a283d2d72dc902d9903b6cc3b68fa0ce3c30b93f506a102",
|
||||
"bytes": 22197
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/17_environmental_contamination_cascade.yaml",
|
||||
"sha256": "556bcaf18251c23f4b0561d7c8d16c5e90e296c3c803e8dfc0f71080a6488904",
|
||||
"bytes": 23054
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/18_genetic_testing_revelation.yaml",
|
||||
"sha256": "a03ec6d38a9005fd566110786ae75583b8fe0382dbd79a82f7ffbfacce0160d5",
|
||||
"bytes": 22923
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/19_employment_fraud_network.yaml",
|
||||
"sha256": "6861efda507da861057e24711255795f389dff0ce2ea66667da2de56ebbb3b10",
|
||||
"bytes": 22985
|
||||
},
|
||||
{
|
||||
"path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/20_medical_malpractice_pattern.yaml",
|
||||
"sha256": "5d986fbcf389d5b8c86844c35184809f77629e53589083663b745ac5a58b467b",
|
||||
"bytes": 23645
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"aggregate": {
|
||||
"advanced_json_cards": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.9,
|
||||
"mean_reward": 0.784375,
|
||||
"hallucination_rate": 0.1
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.6,
|
||||
"mean_reward": 0.7,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.95,
|
||||
"mean_reward": 0.9625,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.8166666666666667,
|
||||
"mean_reward": 0.815625,
|
||||
"hallucination_rate": 0.03333333333333333
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.85,
|
||||
"mean_reward": 0.759375,
|
||||
"hallucination_rate": 0.1
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.8,
|
||||
"mean_reward": 0.85,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.95,
|
||||
"mean_reward": 0.9625,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.8666666666666667,
|
||||
"mean_reward": 0.8572916666666667,
|
||||
"hallucination_rate": 0.03333333333333333
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.95,
|
||||
"mean_reward": 0.834375,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.7,
|
||||
"mean_reward": 0.753125,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.9,
|
||||
"mean_reward": 0.9,
|
||||
"hallucination_rate": 0.1
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.85,
|
||||
"mean_reward": 0.8291666666666667,
|
||||
"hallucination_rate": 0.06666666666666667
|
||||
}
|
||||
},
|
||||
"notes": {
|
||||
"layer1": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.9,
|
||||
"mean_reward": 0.8,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer2": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.65,
|
||||
"mean_reward": 0.721875,
|
||||
"hallucination_rate": 0.05
|
||||
},
|
||||
"layer3": {
|
||||
"n": 20,
|
||||
"pass_rate": 0.85,
|
||||
"mean_reward": 0.8875,
|
||||
"hallucination_rate": 0.0
|
||||
},
|
||||
"overall": {
|
||||
"n": 60,
|
||||
"pass_rate": 0.8,
|
||||
"mean_reward": 0.803125,
|
||||
"hallucination_rate": 0.03333333333333333
|
||||
}
|
||||
}
|
||||
},
|
||||
"token_usage": {
|
||||
"prompt_tokens": 2764882,
|
||||
"completion_tokens": 2000654,
|
||||
"total_tokens": 4765536
|
||||
},
|
||||
"by_mode": {
|
||||
"notes": {
|
||||
"api_calls": 260,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 662861,
|
||||
"completion_tokens": 438410,
|
||||
"total_tokens": 1101271
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 9920610.951999996,
|
||||
"mean_per_call": 38156.19596923077
|
||||
}
|
||||
},
|
||||
"enhanced_notes": {
|
||||
"api_calls": 260,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 635218,
|
||||
"completion_tokens": 352250,
|
||||
"total_tokens": 987468
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 8365401.234999998,
|
||||
"mean_per_call": 32174.620134615383
|
||||
}
|
||||
},
|
||||
"json_cards": {
|
||||
"api_calls": 261,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 657007,
|
||||
"completion_tokens": 469100,
|
||||
"total_tokens": 1126107
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 10583750.727999993,
|
||||
"mean_per_call": 40550.76907279694
|
||||
}
|
||||
},
|
||||
"advanced_json_cards": {
|
||||
"api_calls": 261,
|
||||
"token_usage": {
|
||||
"prompt_tokens": 809796,
|
||||
"completion_tokens": 740894,
|
||||
"total_tokens": 1550690
|
||||
},
|
||||
"latency_ms": {
|
||||
"total": 16943773.900000006,
|
||||
"mean_per_call": 64918.673946360155
|
||||
}
|
||||
}
|
||||
},
|
||||
"api_calls": 1042,
|
||||
"errors": 0
|
||||
},
|
||||
"acceptance": {
|
||||
"all_60_cases": true,
|
||||
"twenty_per_layer": true,
|
||||
"all_four_modes": true,
|
||||
"sequential_memory_only": true,
|
||||
"independent_llm_judge": true,
|
||||
"all_calls_succeeded": true,
|
||||
"passed": true
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user