liqiang
b119135836
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
432 lines
18 KiB
JSON
432 lines
18 KiB
JSON
{
|
|
"schema_version": "2.0",
|
|
"experiment": "5-3",
|
|
"generated_at_utc": "2026-07-29T18:33:57.706070+00:00",
|
|
"protocol": {
|
|
"path": "experiment_protocol.json",
|
|
"sha256": "1c97333e9df04680b1328b482bfc916c330979013c4af2b1974280914f48b19b",
|
|
"content": {
|
|
"schema_version": "1.0",
|
|
"experiment": "5-3",
|
|
"manuscript_source": "book/chapter5.md#experiment-5-3",
|
|
"design": "controlled tau-bench airline policy experiment",
|
|
"model": {
|
|
"name": "qwen3:4b",
|
|
"runtime": "Ollama OpenAI-compatible chat.completions",
|
|
"temperature": 0
|
|
},
|
|
"matched_arms": [
|
|
"natural-language policy plus naive execution tool",
|
|
"natural-language policy plus checklist parameters plus server-ground-truth gate"
|
|
],
|
|
"cases": {
|
|
"count": 60,
|
|
"factorial_matrix": {
|
|
"cabin": [
|
|
"basic_economy",
|
|
"economy_flex",
|
|
"business"
|
|
],
|
|
"hours_since_booking": [
|
|
5.0,
|
|
24.0,
|
|
24.1,
|
|
26.0,
|
|
120.0
|
|
],
|
|
"flight_status": [
|
|
"scheduled",
|
|
"cancelled_by_airline",
|
|
"delayed_major",
|
|
"delayed_minor"
|
|
]
|
|
},
|
|
"user_variants": 5
|
|
},
|
|
"metrics": [
|
|
"task success rate",
|
|
"policy violations",
|
|
"invalid tool calls",
|
|
"user experience proxy",
|
|
"checklist expected-value mismatch rate",
|
|
"paired exact McNemar significance"
|
|
],
|
|
"acceptance": {
|
|
"same_real_qwen3_4b_model_both_arms": true,
|
|
"all_60_cases_attempted_in_both_arms": true,
|
|
"server_policy_uses_database_facts_and_server_clock_only": true,
|
|
"raw_messages_tool_calls_provider_receipts_and_usage_saved": true
|
|
}
|
|
}
|
|
},
|
|
"config": {
|
|
"provider": "ollama",
|
|
"small_model": "qwen3:4b",
|
|
"big_model": null,
|
|
"mode": "both",
|
|
"task_ids": [
|
|
"TB001-basic_economy-h5p0-scheduled"
|
|
]
|
|
},
|
|
"arms": [
|
|
{
|
|
"key": "small_control",
|
|
"mode": "control",
|
|
"model": "qwen3:4b",
|
|
"label": "小模型·纯自然语言",
|
|
"role": "控制组",
|
|
"summary": {
|
|
"n": 1,
|
|
"success": 1,
|
|
"success_rate": 1.0,
|
|
"violations": 0,
|
|
"invalid": 0,
|
|
"user_experience_success": 1,
|
|
"user_experience_rate": 1.0,
|
|
"checklist_total": 0,
|
|
"checklist_mismatch": 0
|
|
},
|
|
"results": [
|
|
{
|
|
"task_id": "TB001-basic_economy-h5p0-scheduled",
|
|
"expect_refundable": true,
|
|
"refunded": true,
|
|
"success": true,
|
|
"wrongful_refund": false,
|
|
"wrongful_refusal": false,
|
|
"invalid_tool_calls": 0,
|
|
"offered_alt": null,
|
|
"explained_policy": true,
|
|
"user_experience_ok": true,
|
|
"checklist_records": [],
|
|
"source": {
|
|
"design": "controlled tau-bench airline policy matrix",
|
|
"cabin": "basic_economy",
|
|
"hours_since_booking": 5.0,
|
|
"flight_status": "scheduled",
|
|
"user_variant": 0
|
|
},
|
|
"final_text": "The reservation TB001 has been successfully canceled, and the full refund of **501.00 yuan** has been processed to your original payment method. \n\nThis refund is allowed under the policy because your booking was made within 24 hours (5 hours ago) for a basic_economy ticket. No additional action is required — the refund will appear in your account within 3-5 business days. \n\nThank you for your patience! ✈️",
|
|
"transcript": [
|
|
{
|
|
"tool": "get_reservation",
|
|
"args": {
|
|
"reservation_id": "TB001"
|
|
},
|
|
"result": {
|
|
"status": "ok",
|
|
"reservation_id": "TB001",
|
|
"passenger_name": "Passenger 001",
|
|
"flight_no": "TAU001",
|
|
"route": "SFO-JFK",
|
|
"depart_date": "2026-09-01",
|
|
"cabin": "basic_economy",
|
|
"price": 501.0,
|
|
"reservation_status": "active",
|
|
"flight_status": "scheduled",
|
|
"server_time": "2026-07-17T12:00:00",
|
|
"booked_at": "2026-07-17T07:00:00",
|
|
"hours_since_booking": 5.0
|
|
}
|
|
},
|
|
{
|
|
"tool": "cancel_reservation",
|
|
"args": {
|
|
"reservation_id": "TB001"
|
|
},
|
|
"result": {
|
|
"status": "ok",
|
|
"message": "预订 TB001 已取消，全额退款 501.0 元已原路退回。",
|
|
"refund_amount": 501.0
|
|
}
|
|
}
|
|
],
|
|
"messages": [
|
|
{
|
|
"role": "system",
|
|
"content": "你是 SkyWing 航空公司的客服 Agent，可以帮乘客查询和取消预订。\n\n【退款政策（自然语言）】\n- 经济舱基础票（basic_economy）默认不可退款。\n- 例外 1：下单后 24 小时内，可全额退款。\n- 例外 2：航班被航司取消，或发生重大延误（延误 ≥ 3 小时），可全额退款。\n- 经济舱灵活票（economy_flex）与商务舱（business）可全额退款。\n- 若不可退款：应礼貌解释政策，并主动提议替代方案（如保留客票改签、申请旅行信用点），\n 绝不能给用户退款。\n\n请先判断能否退款，再决定是否调用取消/退款工具。乘客自述的信息（舱位、下单时间等）\n可能不准确，应以系统查询到的预订信息为准。"
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": "Please cancel reservation TB001 and issue the refund allowed by policy."
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_erpwan7b",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "get_reservation",
|
|
"arguments": "{\"reservation_id\":\"TB001\"}"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": "call_erpwan7b",
|
|
"content": "{\"status\": \"ok\", \"reservation_id\": \"TB001\", \"passenger_name\": \"Passenger 001\", \"flight_no\": \"TAU001\", \"route\": \"SFO-JFK\", \"depart_date\": \"2026-09-01\", \"cabin\": \"basic_economy\", \"price\": 501.0, \"reservation_status\": \"active\", \"flight_status\": \"scheduled\", \"server_time\": \"2026-07-17T12:00:00\", \"booked_at\": \"2026-07-17T07:00:00\", \"hours_since_booking\": 5.0}"
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_lnp7jhzj",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "cancel_reservation",
|
|
"arguments": "{\"reservation_id\":\"TB001\"}"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": "call_lnp7jhzj",
|
|
"content": "{\"status\": \"ok\", \"message\": \"预订 TB001 已取消，全额退款 501.0 元已原路退回。\", \"refund_amount\": 501.0}"
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": "The reservation TB001 has been successfully canceled, and the full refund of **501.00 yuan** has been processed to your original payment method. \n\nThis refund is allowed under the policy because your booking was made within 24 hours (5 hours ago) for a basic_economy ticket. No additional action is required — the refund will appear in your account within 3-5 business days. \n\nThank you for your patience! ✈️"
|
|
}
|
|
],
|
|
"provider_receipts": [
|
|
{
|
|
"turn": 1,
|
|
"response_id": "chatcmpl-459",
|
|
"response_model": "qwen3:4b",
|
|
"finish_reason": "tool_calls",
|
|
"usage": {
|
|
"prompt_tokens": 422,
|
|
"completion_tokens": 573,
|
|
"total_tokens": 995,
|
|
"cached_prompt_tokens": null
|
|
}
|
|
},
|
|
{
|
|
"turn": 2,
|
|
"response_id": "chatcmpl-696",
|
|
"response_model": "qwen3:4b",
|
|
"finish_reason": "tool_calls",
|
|
"usage": {
|
|
"prompt_tokens": 616,
|
|
"completion_tokens": 1024,
|
|
"total_tokens": 1640,
|
|
"cached_prompt_tokens": null
|
|
}
|
|
},
|
|
{
|
|
"turn": 3,
|
|
"response_id": "chatcmpl-959",
|
|
"response_model": "qwen3:4b",
|
|
"finish_reason": "stop",
|
|
"usage": {
|
|
"prompt_tokens": 697,
|
|
"completion_tokens": 456,
|
|
"total_tokens": 1153,
|
|
"cached_prompt_tokens": null
|
|
}
|
|
}
|
|
],
|
|
"duration_s": 34.697
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"key": "small_codified",
|
|
"mode": "codified",
|
|
"model": "qwen3:4b",
|
|
"label": "小模型+代码化规则",
|
|
"role": "实验组",
|
|
"summary": {
|
|
"n": 1,
|
|
"success": 1,
|
|
"success_rate": 1.0,
|
|
"violations": 0,
|
|
"invalid": 0,
|
|
"user_experience_success": 1,
|
|
"user_experience_rate": 1.0,
|
|
"checklist_total": 1,
|
|
"checklist_mismatch": 0
|
|
},
|
|
"results": [
|
|
{
|
|
"task_id": "TB001-basic_economy-h5p0-scheduled",
|
|
"expect_refundable": true,
|
|
"refunded": true,
|
|
"success": true,
|
|
"wrongful_refund": false,
|
|
"wrongful_refusal": false,
|
|
"invalid_tool_calls": 0,
|
|
"offered_alt": null,
|
|
"explained_policy": true,
|
|
"user_experience_ok": true,
|
|
"checklist_records": [
|
|
{
|
|
"reservation_id": "TB001",
|
|
"expected_refundable": true,
|
|
"actual_refundable": true,
|
|
"match": true,
|
|
"actual_reason": "within_24h",
|
|
"expected_reason": "within_24h"
|
|
}
|
|
],
|
|
"source": {
|
|
"design": "controlled tau-bench airline policy matrix",
|
|
"cabin": "basic_economy",
|
|
"hours_since_booking": 5.0,
|
|
"flight_status": "scheduled",
|
|
"user_variant": 0
|
|
},
|
|
"final_text": "The reservation TB001 has been successfully canceled and a full refund of **501.0 yuan** has been issued. This was approved under the policy exception for bookings made within 24 hours (your reservation was booked 5 hours ago). \n\n✅ **Refund Details**: \n- Refund Amount: 501.0 yuan \n- Reason: within_24h (policy exception) \n- Status: Confirmed by system (db_truth matches your checklist) \n\nNo further action is required. Thank you for your patience! ✈️",
|
|
"transcript": [
|
|
{
|
|
"tool": "get_reservation",
|
|
"args": {
|
|
"reservation_id": "TB001"
|
|
},
|
|
"result": {
|
|
"status": "ok",
|
|
"reservation_id": "TB001",
|
|
"passenger_name": "Passenger 001",
|
|
"flight_no": "TAU001",
|
|
"route": "SFO-JFK",
|
|
"depart_date": "2026-09-01",
|
|
"cabin": "basic_economy",
|
|
"price": 501.0,
|
|
"reservation_status": "active",
|
|
"flight_status": "scheduled",
|
|
"server_time": "2026-07-17T12:00:00",
|
|
"booked_at": "2026-07-17T07:00:00",
|
|
"hours_since_booking": 5.0
|
|
}
|
|
},
|
|
{
|
|
"tool": "cancel_reservation",
|
|
"args": {
|
|
"reservation_id": "TB001",
|
|
"expected_refundable": true,
|
|
"expected_reason": "within_24h"
|
|
},
|
|
"result": {
|
|
"status": "ok",
|
|
"message": "已按数据库真值校验通过（within_24h）。预订 TB001 已取消，全额退款 501.0 元。",
|
|
"refund_amount": 501.0,
|
|
"db_truth": {
|
|
"refundable": true,
|
|
"reason": "within_24h"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"messages": [
|
|
{
|
|
"role": "system",
|
|
"content": "你是 SkyWing 航空公司的客服 Agent，可以帮乘客查询和取消预订。\n\n【退款政策（自然语言）】\n- 经济舱基础票（basic_economy）默认不可退款。\n- 例外 1：下单后 24 小时内，可全额退款。\n- 例外 2：航班被航司取消，或发生重大延误（延误 ≥ 3 小时），可全额退款。\n- 经济舱灵活票（economy_flex）与商务舱（business）可全额退款。\n- 若不可退款：应礼貌解释政策，并主动提议替代方案（如保留客票改签、申请旅行信用点），\n 绝不能给用户退款。\n\n请先判断能否退款，再决定是否调用取消/退款工具。乘客自述的信息（舱位、下单时间等）\n可能不准确，应以系统查询到的预订信息为准。\n\n【操作要求】\n调用 cancel_reservation 前，请先用 get_reservation 查询真实预订信息，逐条核对退款政策，\n并在 expected_refundable / expected_reason 参数中如实填写你的判断（这是一份调用前 checklist）。\n系统会以数据库真值为准进行校验：若你的判断与真值不符或存在违规，调用会被拒绝。"
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": "Please cancel reservation TB001 and issue the refund allowed by policy."
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_wmhjlrrk",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "get_reservation",
|
|
"arguments": "{\"reservation_id\":\"TB001\"}"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": "call_wmhjlrrk",
|
|
"content": "{\"status\": \"ok\", \"reservation_id\": \"TB001\", \"passenger_name\": \"Passenger 001\", \"flight_no\": \"TAU001\", \"route\": \"SFO-JFK\", \"depart_date\": \"2026-09-01\", \"cabin\": \"basic_economy\", \"price\": 501.0, \"reservation_status\": \"active\", \"flight_status\": \"scheduled\", \"server_time\": \"2026-07-17T12:00:00\", \"booked_at\": \"2026-07-17T07:00:00\", \"hours_since_booking\": 5.0}"
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_mx5v341w",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "cancel_reservation",
|
|
"arguments": "{\"reservation_id\":\"TB001\",\"expected_refundable\":true,\"expected_reason\":\"within_24h\"}"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": "call_mx5v341w",
|
|
"content": "{\"status\": \"ok\", \"message\": \"已按数据库真值校验通过（within_24h）。预订 TB001 已取消，全额退款 501.0 元。\", \"refund_amount\": 501.0, \"db_truth\": {\"refundable\": true, \"reason\": \"within_24h\"}}"
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": "The reservation TB001 has been successfully canceled and a full refund of **501.0 yuan** has been issued. This was approved under the policy exception for bookings made within 24 hours (your reservation was booked 5 hours ago). \n\n✅ **Refund Details**: \n- Refund Amount: 501.0 yuan \n- Reason: within_24h (policy exception) \n- Status: Confirmed by system (db_truth matches your checklist) \n\nNo further action is required. Thank you for your patience! ✈️"
|
|
}
|
|
],
|
|
"provider_receipts": [
|
|
{
|
|
"turn": 1,
|
|
"response_id": "chatcmpl-730",
|
|
"response_model": "qwen3:4b",
|
|
"finish_reason": "tool_calls",
|
|
"usage": {
|
|
"prompt_tokens": 735,
|
|
"completion_tokens": 308,
|
|
"total_tokens": 1043,
|
|
"cached_prompt_tokens": null
|
|
}
|
|
},
|
|
{
|
|
"turn": 2,
|
|
"response_id": "chatcmpl-261",
|
|
"response_model": "qwen3:4b",
|
|
"finish_reason": "tool_calls",
|
|
"usage": {
|
|
"prompt_tokens": 929,
|
|
"completion_tokens": 578,
|
|
"total_tokens": 1507,
|
|
"cached_prompt_tokens": null
|
|
}
|
|
},
|
|
{
|
|
"turn": 3,
|
|
"response_id": "chatcmpl-507",
|
|
"response_model": "qwen3:4b",
|
|
"finish_reason": "stop",
|
|
"usage": {
|
|
"prompt_tokens": 1057,
|
|
"completion_tokens": 482,
|
|
"total_tokens": 1539,
|
|
"cached_prompt_tokens": null
|
|
}
|
|
}
|
|
],
|
|
"duration_s": 20.097
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"paired_analysis": {
|
|
"test": "two-sided exact McNemar/binomial test",
|
|
"n": 1,
|
|
"control_only": 0,
|
|
"codified_only": 0,
|
|
"discordant": 0,
|
|
"p_value": 1.0,
|
|
"control_success_rate": 1.0,
|
|
"codified_success_rate": 1.0,
|
|
"success_rate_delta": 0.0,
|
|
"codified_significantly_higher": false
|
|
}
|
|
} |