{
  "schema_version": "chapter3-evidence-v1",
  "experiment": "3-9",
  "run_id": "20260729T214519Z-3_9-1bbf4623",
  "created_at": "2026-07-29T21:45:19.517360+00:00",
  "status": "passed",
  "run_dir": "/Users/boj/book/ai-agent-book/chapter3/agentic-rag-for-user-memory/validation/runs/20260729T214519Z-3_9-1bbf4623",
  "artifacts": {
    "evidence.json": "569c72db7e04eae6f2354b82c4c21c6d83f13a738839d01b64932ec9e0c69e1d",
    "receipts.json": "25fea7edfb86bc80377160464bd4323559238eb68a3af82fec1df4f9d2d64389",
    "manifest.json": "a935c7542b6830a078ee70d78891cf1981f9ee70d0d41c661fd4d95cce9e817c"
  },
  "inputs": [
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/memory_rag_campaign.py",
      "sha256": "61de7c33037b30c438f3e1411d7c4ddbd55ade162f92f444d5781234424c21ba",
      "bytes": 29618
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/agentic-rag-for-user-memory/campaign.py",
      "sha256": "36b917d64e02aa3a75bbe7c323e9dfcb8d4db1a6b6fe2b8f34a6bd20e0b55d01",
      "bytes": 294
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/01_bank_account_setup.yaml",
      "sha256": "e9f788fa393a0f07c83f32e90f05d0b2212355ba62e0a5c3288a469b5feb75b6",
      "bytes": 11396
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/02_insurance_claim.yaml",
      "sha256": "ba4d78069e6aaedc1876cbbd3069250ebeafd1def4b619463218ad3a33f80119",
      "bytes": 13222
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/03_medical_appointment.yaml",
      "sha256": "42877892ab8720090853ac0fa9050e4097df967a4d88893e25563766e536d3bb",
      "bytes": 12942
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/04_airline_booking.yaml",
      "sha256": "76b354c62739d5a23cc71fd94c7a16a0c9552fe8014bc9ddafe262ea0df366de",
      "bytes": 11836
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/05_internet_service_setup.yaml",
      "sha256": "f5836b44ae54ee6c220b0261fce73f27e211ed5436b8a52afad17ba6c399658d",
      "bytes": 12283
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/06_credit_card_application.yaml",
      "sha256": "7c4358326df8671cad9814a8796df48e5a60cd807ee89a33e6d8523302e8630e",
      "bytes": 11172
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/07_car_rental.yaml",
      "sha256": "e497e5819e228f25c58a6f40a29d9d56931eef19c032480ee09eb247a483ebea",
      "bytes": 10705
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/08_hotel_reservation.yaml",
      "sha256": "16ddb69e00d880376e869fa454af0b697301c24d7b6b0be20e090ebd9ef64c20",
      "bytes": 10229
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/09_home_security.yaml",
      "sha256": "50bc194260c94f8f4e0dd773e5a07dc58d0bf47b72df40aa4a1bc2f7adf5ab3c",
      "bytes": 11064
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/10_pharmacy_transfer.yaml",
      "sha256": "d7051d7da1e99b8c662c02cf4bf8a790e72c82efc684562daeb6714f6cd67279",
      "bytes": 9554
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/11_mortgage_application.yaml",
      "sha256": "b34269d9b1adc69dc1579317215e5e410c3f85f9f688b5ac283bd419df91a006",
      "bytes": 14009
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/12_gym_membership.yaml",
      "sha256": "c530e2215951be9f6553c3ee0c29e68ea596bd9215fa72deda13a44da8ff4d79",
      "bytes": 12243
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/13_tax_preparation.yaml",
      "sha256": "ce2af37f2ace66b65cd0f834b8fa727abd7f5399ae260c43c26d07f00bd20e03",
      "bytes": 14382
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/14_cellphone_upgrade.yaml",
      "sha256": "26fbf77437deaec9d9bfc635700d3a9f688904d8db594d47b337a2db3de25d83",
      "bytes": 11402
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/15_college_enrollment.yaml",
      "sha256": "4252cded12f1e5821d3ad3cbbe270dc4c526cf6ffb86e53bea6db163a3e4f847",
      "bytes": 12507
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/16_home_renovation.yaml",
      "sha256": "e66f44d9b3d632cf2afc2759386e8e81dc0edb89b735450fcf5e2e54e2db24fc",
      "bytes": 11383
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/17_veterinary_care.yaml",
      "sha256": "712b5726f9d3732d915cb338bf7080d5c048f1156e0718434befa2b86ae92013",
      "bytes": 10538
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/18_retirement_planning.yaml",
      "sha256": "46e3876a7d84c70b2bba668f2454991d15066628e83cce97e5fb6abd2ba6558f",
      "bytes": 10650
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/19_wedding_venue.yaml",
      "sha256": "b2066bf3897c9b022b63321d9499a254d4b709069b1f5e5d9a0a69dc44026493",
      "bytes": 10141
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer1/20_daycare_enrollment.yaml",
      "sha256": "d6d6a8e6c1130fd693f7aa2febc8e40b49df4848f2ed45ccc849dc78432f315e",
      "bytes": 11030
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/01_multiple_vehicles.yaml",
      "sha256": "d00b83e6db7660a2686606218e6b592af883aea201b066a3f6e769d7710e4b48",
      "bytes": 19514
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/02_multiple_properties.yaml",
      "sha256": "21b6b80b63eb36481c12317ed63be09d033bcaf7e2526e8abf91d2bdd70d8451",
      "bytes": 19694
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/03_multiple_credit_cards.yaml",
      "sha256": "3cde2ffbf8f4f75ed4fc3e1f59c0a1c388582f3223a792e1f496b5ba1b865486",
      "bytes": 23868
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/04_multiple_subscriptions.yaml",
      "sha256": "25e2dab7c6f1e12a9ce72646dd702026b19b15d1c8a76aec5d82be4eb0503298",
      "bytes": 22110
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/05_multiple_bank_accounts.yaml",
      "sha256": "b30b4c9e503737bd600957e1f3783d046930263ecb78c970056e63b25ee63cc6",
      "bytes": 21604
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/06_multiple_insurance_policies.yaml",
      "sha256": "28ec764bd484bb7856941168dc7e557c4eaa791fe54e6b0e466dd6b0ab7776ea",
      "bytes": 22244
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/07_multiple_medications.yaml",
      "sha256": "abaa76f5e90ddf3f1a1af1e5bb75fba0b3fae3d77fceb86c54da0d2584c50c24",
      "bytes": 27025
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/08_multiple_rental_properties.yaml",
      "sha256": "5cc7d58328014fe15e3395d201ebbb57c8dbd0d6f51698064aed974f9ead58bd",
      "bytes": 33146
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/09_multiple_children_schools.yaml",
      "sha256": "9a0f2647512cfdea680623de4ccd78e3a91520d6471d6905e822f29b05a3e3ea",
      "bytes": 23765
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/10_travel_rebooking_chain.yaml",
      "sha256": "e3d1a0907e33311b6123c25a4890449572edebc99b83c691244e5df850488e17",
      "bytes": 22888
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/11_medical_treatment_evolution.yaml",
      "sha256": "32703b7e705fbb5721d9db74991b6662951dae7d4a5e99bdc943e1f5183f2b87",
      "bytes": 23389
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/12_contradictory_financial_instructions.yaml",
      "sha256": "e2842813485c1684d35371c77c9a439fb8b064febcd15f7210a06856da73b596",
      "bytes": 21736
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/13_home_services_cascade.yaml",
      "sha256": "c59d7acce9b2b4571d0128e25ffe7813bd60105ec0aebe8cd6a81f55b99a9f1f",
      "bytes": 21586
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/14_product_order_modifications.yaml",
      "sha256": "6d4fb1795b6c2752849f26d82a614aa0358a5bab16dd3c651baa86d3bb6a7017",
      "bytes": 20856
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/15_employment_negotiation.yaml",
      "sha256": "d98ce6a5e4fc8758aa011fd98f1016793d89c62f23743b95fb939d1619c71aa8",
      "bytes": 21116
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/16_family_event_conflicting_input.yaml",
      "sha256": "e54fe6ccf2de79135778d0cd9391cb2da1ed3ac731b252da9fbdf5b6ab7583a0",
      "bytes": 22082
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/17_tech_support_cascade.yaml",
      "sha256": "07c21fd2c800681e6957f9b0d83a23bd931bb60e3d3d5c3303517fd216c6229c",
      "bytes": 20478
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/18_education_prerequisite_chain.yaml",
      "sha256": "698dd0d18011559ff242911fec35f97c7d42c851ae02183dbb37ff6c81aec324",
      "bytes": 20588
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/19_investment_market_response.yaml",
      "sha256": "1fbfec4ea12194673b8abb3e0f81a0414021536874b5837195a4fec849c9f53b",
      "bytes": 20608
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer2/20_healthcare_coverage_changes.yaml",
      "sha256": "edebb2a0efdf000103e292f305b7aa04600555118fe28a1a61c645585174c76b",
      "bytes": 22201
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/01_travel_coordination.yaml",
      "sha256": "6d4ec28d1a5e9ec16d1f8c6367de90827d762b0449fc457c8731b79e29222831",
      "bytes": 24508
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/02_medical_insurance_coordination.yaml",
      "sha256": "f4ea8d642ac182431efa9ce70d62985dbfe1ac17e3b980a63aad975fea7b561d",
      "bytes": 25280
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/03_home_purchase_coordination.yaml",
      "sha256": "df68870ca4eb344c5a4da6d8f9275ca59fef6edb1ab0a67381e409f656027c33",
      "bytes": 26717
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/04_warranty_coordination.yaml",
      "sha256": "08501b6528d39e67e8278720d3b52d26c67e5f03304d82ca17478d85a5183208",
      "bytes": 27861
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/05_tax_preparation_synthesis.yaml",
      "sha256": "63a33174166e03d8e747f7ac9b831dcc366b8b70ba7431ac00c6a1004fc569b2",
      "bytes": 28662
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/06_business_expansion_coordination.yaml",
      "sha256": "d0b2af52fa1b1e9209d90d6f551ed36d8ca69faf0e58de61f982bc893228baaa",
      "bytes": 29714
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/07_eldercare_coordination.yaml",
      "sha256": "71c066861653baac39541b16efd4deae4bad5b275e3260069a0570c8b0df7296",
      "bytes": 32566
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/08_divorce_settlement_complexity.yaml",
      "sha256": "1e8ae2d505d5bf50321538bca47e5799b819f749dc932544a94ed812116c0d2f",
      "bytes": 33565
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/09_vehicle_accident_cascade.yaml",
      "sha256": "29234d6a933174a9e8b0798e183abe4b60b3edb42bace735c77f6fb115de480e",
      "bytes": 29687
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/10_education_financing_maze.yaml",
      "sha256": "7c5fdc9050dfb34c4c3c90d59f9ebfe209e8f3ac93373a34c95a8e1bcc1119fd",
      "bytes": 29533
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/11_immigration_status_complexity.yaml",
      "sha256": "15bcd4aeae39f45f354c192b38c77f6071e53579ff6e92e72917bc87dae0a7c4",
      "bytes": 25066
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/12_real_estate_investment_tangle.yaml",
      "sha256": "07535fcd40deb5c80d23e5d2faf821b36fcbca42402e01ecd985d5d70545af99",
      "bytes": 27421
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/13_emergency_medical_cascade.yaml",
      "sha256": "2a488619f9071ad79691b262db7c2a7087e039897138178a7d9ac7536acc8524",
      "bytes": 22078
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/14_hidden_medical_insurance_web.yaml",
      "sha256": "3e8492958ed80ae8aaf85a3ee814fdb5ea6615f4f264073d8fb194add1c6a0b5",
      "bytes": 24624
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/15_identity_theft_discovery.yaml",
      "sha256": "27a100b2b13c2fe93f40ac85f0927a0489953b0e4423d78db342b4e48891179a",
      "bytes": 23071
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/16_cryptocurrency_inheritance_puzzle.yaml",
      "sha256": "4179a9c39ca7aa1c8a283d2d72dc902d9903b6cc3b68fa0ce3c30b93f506a102",
      "bytes": 22197
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/17_environmental_contamination_cascade.yaml",
      "sha256": "556bcaf18251c23f4b0561d7c8d16c5e90e296c3c803e8dfc0f71080a6488904",
      "bytes": 23054
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/18_genetic_testing_revelation.yaml",
      "sha256": "a03ec6d38a9005fd566110786ae75583b8fe0382dbd79a82f7ffbfacce0160d5",
      "bytes": 22923
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/19_employment_fraud_network.yaml",
      "sha256": "6861efda507da861057e24711255795f389dff0ce2ea66667da2de56ebbb3b10",
      "bytes": 22985
    },
    {
      "path": "/Users/boj/book/ai-agent-book/chapter3/user-memory-evaluation/test_cases/layer3/20_medical_malpractice_pattern.yaml",
      "sha256": "5d986fbcf389d5b8c86844c35184809f77629e53589083663b745ac5a58b467b",
      "bytes": 23645
    }
  ],
  "summary": {
    "aggregate": {
      "plain": {
        "layer1": {
          "n": 20,
          "pass_rate": 0.95,
          "mean_reward": 0.86875,
          "hallucination_rate": 0.05
        },
        "layer2": {
          "n": 20,
          "pass_rate": 0.75,
          "mean_reward": 0.73125,
          "hallucination_rate": 0.1
        },
        "layer3": {
          "n": 20,
          "pass_rate": 0.65,
          "mean_reward": 0.725,
          "hallucination_rate": 0.1
        },
        "overall": {
          "n": 60,
          "pass_rate": 0.7833333333333333,
          "mean_reward": 0.775,
          "hallucination_rate": 0.08333333333333333
        }
      }
    },
    "api_calls": 253,
    "token_usage": {
      "prompt_tokens": 804972,
      "completion_tokens": 136007,
      "total_tokens": 940979
    },
    "errors": 0,
    "resumed_cases": 60
  },
  "acceptance": {
    "all_60_cases": true,
    "twenty_per_layer": true,
    "fixed_window_indexing": true,
    "live_agent_generated_searches": true,
    "raw_chunks_and_trajectories_retained": true,
    "independent_external_judge": true,
    "raw_request_response_receipts": true,
    "all_calls_succeeded": true,
    "same_three_layer_judge_metrics": true,
    "agent_can_iterate": true,
    "canonical_receipts_cover_planner_answer_and_joint_judge": true,
    "passed": true
  }
}