liqiang
b119135836
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
634 lines
14 KiB
JSON
634 lines
14 KiB
JSON
[
|
|
{
|
|
"task": "forms-insurance",
|
|
"condition": "duplex",
|
|
"wall_time": 274.9,
|
|
"error": null,
|
|
"n_user_turns": 9,
|
|
"n_slow_steps": 30,
|
|
"n_to_slow": 10,
|
|
"n_ask_user": 4,
|
|
"voice_latency_p50": 2.451,
|
|
"voice_latency_p90": 2.854,
|
|
"voice_latency_mean": 2.587,
|
|
"n_latency_samples": 9,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"full_name": "Sarah Mitchell",
|
|
"dob": "1989-04-12",
|
|
"email": "sarah.mitchell89@gmail.com",
|
|
"phone": "415-555-0173",
|
|
"street": "2847 Fillmore Street",
|
|
"city": "San Francisco",
|
|
"zip": "94123",
|
|
"vehicle_make": "Toyota",
|
|
"vehicle_model": "RAV4",
|
|
"vehicle_year": "2021",
|
|
"coverage": "full",
|
|
"prior_claims": "no"
|
|
},
|
|
"probe_correct": null
|
|
},
|
|
{
|
|
"task": "forms-insurance",
|
|
"condition": "duplex",
|
|
"wall_time": 321.8,
|
|
"error": null,
|
|
"n_user_turns": 9,
|
|
"n_slow_steps": 33,
|
|
"n_to_slow": 8,
|
|
"n_ask_user": 5,
|
|
"voice_latency_p50": 2.39,
|
|
"voice_latency_p90": 2.863,
|
|
"voice_latency_mean": 2.53,
|
|
"n_latency_samples": 9,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"full_name": "Sarah Mitchell",
|
|
"dob": "1989-04-12",
|
|
"email": "sarah.mitchell89@gmail.com",
|
|
"phone": "415-555-0173",
|
|
"street": "2847 Fillmore Street",
|
|
"city": "San Francisco",
|
|
"zip": "94123",
|
|
"vehicle_make": "Toyota",
|
|
"vehicle_model": "RAV4",
|
|
"vehicle_year": "2021",
|
|
"coverage": "full",
|
|
"prior_claims": "no"
|
|
},
|
|
"probe_correct": null
|
|
},
|
|
{
|
|
"task": "forms-insurance",
|
|
"condition": "strawman",
|
|
"wall_time": 225.9,
|
|
"error": null,
|
|
"n_user_turns": 6,
|
|
"n_slow_steps": 31,
|
|
"n_to_slow": 0,
|
|
"n_ask_user": 6,
|
|
"voice_latency_p50": 18.296,
|
|
"voice_latency_p90": 22.962,
|
|
"voice_latency_mean": 18.505,
|
|
"n_latency_samples": 7,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"full_name": "Sarah Mitchell",
|
|
"dob": "1989-04-12",
|
|
"email": "sarah.mitchell89@gmail.com",
|
|
"phone": "415-555-0173",
|
|
"street": "2847 Fillmore Street",
|
|
"city": "San Francisco",
|
|
"zip": "94123",
|
|
"vehicle_make": "Toyota",
|
|
"vehicle_model": "RAV4",
|
|
"vehicle_year": "2021",
|
|
"coverage": "full",
|
|
"prior_claims": "no"
|
|
},
|
|
"probe_correct": null
|
|
},
|
|
{
|
|
"task": "forms-insurance",
|
|
"condition": "strawman",
|
|
"wall_time": 247.3,
|
|
"error": null,
|
|
"n_user_turns": 10,
|
|
"n_slow_steps": 36,
|
|
"n_to_slow": 0,
|
|
"n_ask_user": 7,
|
|
"voice_latency_p50": 10.862,
|
|
"voice_latency_p90": 12.52,
|
|
"voice_latency_mean": 10.134,
|
|
"n_latency_samples": 11,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"full_name": "Sarah Mitchell",
|
|
"dob": "1989-04-12",
|
|
"email": "sarah.mitchell89@gmail.com",
|
|
"phone": "415-555-0173",
|
|
"street": "2847 Fillmore Street",
|
|
"city": "San Francisco",
|
|
"zip": "94123",
|
|
"vehicle_make": "Toyota",
|
|
"vehicle_model": "RAV4",
|
|
"vehicle_year": "2021",
|
|
"coverage": "full",
|
|
"prior_claims": "no"
|
|
},
|
|
"probe_correct": null
|
|
},
|
|
{
|
|
"task": "booking-flight",
|
|
"condition": "duplex",
|
|
"wall_time": 239.0,
|
|
"error": null,
|
|
"n_user_turns": 6,
|
|
"n_slow_steps": 22,
|
|
"n_to_slow": 6,
|
|
"n_ask_user": 3,
|
|
"voice_latency_p50": 1.847,
|
|
"voice_latency_p90": 2.039,
|
|
"voice_latency_mean": 1.857,
|
|
"n_latency_samples": 6,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"flight": {
|
|
"airline": "Cirrus",
|
|
"arrive": "13:05",
|
|
"date": "2026-07-21",
|
|
"depart": "09:45",
|
|
"from": "SFO",
|
|
"id": "FL110",
|
|
"price": 385,
|
|
"stops": 0,
|
|
"to": "JFK"
|
|
},
|
|
"passenger": "Daniel Ortiz",
|
|
"checks": {
|
|
"date": true,
|
|
"depart": true,
|
|
"stops": true,
|
|
"price": true,
|
|
"passenger": true
|
|
}
|
|
},
|
|
"probe_correct": 0.5,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "wrong",
|
|
"evidence": "Agent said only two nonstop morning flights (FL110 $385, Delta $420); gold says there are 4 nonstop options total. Agent incorrectly claimed those are the only nonstop flights."
|
|
},
|
|
{
|
|
"q": 1,
|
|
"grade": "correct",
|
|
"evidence": "'the total is $385 for your flight'"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "booking-flight",
|
|
"condition": "duplex",
|
|
"wall_time": 201.8,
|
|
"error": null,
|
|
"n_user_turns": 5,
|
|
"n_slow_steps": 20,
|
|
"n_to_slow": 6,
|
|
"n_ask_user": 3,
|
|
"voice_latency_p50": 1.605,
|
|
"voice_latency_p90": 1.887,
|
|
"voice_latency_mean": 1.71,
|
|
"n_latency_samples": 5,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"flight": {
|
|
"airline": "Cirrus",
|
|
"arrive": "13:05",
|
|
"date": "2026-07-21",
|
|
"depart": "09:45",
|
|
"from": "SFO",
|
|
"id": "FL110",
|
|
"price": 385,
|
|
"stops": 0,
|
|
"to": "JFK"
|
|
},
|
|
"passenger": "Daniel Ortiz",
|
|
"checks": {
|
|
"date": true,
|
|
"depart": true,
|
|
"stops": true,
|
|
"price": true,
|
|
"passenger": true
|
|
}
|
|
},
|
|
"probe_correct": 0.5,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "wrong",
|
|
"evidence": "Agent said 'three options' for nonstop morning flights under $400, but the gold says 4 nonstop options total (06:15 $420, 09:45 $385, 13:00 $455, 17:25 $340). Agent missed the 17:25 $340 option and framed the count incorrectly."
|
|
},
|
|
{
|
|
"q": 1,
|
|
"grade": "correct",
|
|
"evidence": "'yes, three eighty five is the final price' - matches gold $385."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "booking-flight",
|
|
"condition": "strawman",
|
|
"wall_time": 235.6,
|
|
"error": null,
|
|
"n_user_turns": 5,
|
|
"n_slow_steps": 25,
|
|
"n_to_slow": 0,
|
|
"n_ask_user": 3,
|
|
"voice_latency_p50": 19.739,
|
|
"voice_latency_p90": 21.142,
|
|
"voice_latency_mean": 18.185,
|
|
"n_latency_samples": 6,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"flight": {
|
|
"airline": "Cirrus",
|
|
"arrive": "13:05",
|
|
"date": "2026-07-21",
|
|
"depart": "09:45",
|
|
"from": "SFO",
|
|
"id": "FL110",
|
|
"price": 385,
|
|
"stops": 0,
|
|
"to": "JFK"
|
|
},
|
|
"passenger": "Daniel Ortiz",
|
|
"checks": {
|
|
"date": true,
|
|
"depart": true,
|
|
"stops": true,
|
|
"price": true,
|
|
"passenger": true
|
|
}
|
|
},
|
|
"probe_correct": 1.0,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "correct",
|
|
"evidence": "There are four nonstops: FL108 Delta at 06:15 for $420, FL110 Cirrus at 09:45 for $385, FL112 Blue Sky at 13:00 for $455, and FL114 Acme at 17:25 for $340."
|
|
},
|
|
{
|
|
"q": 1,
|
|
"grade": "correct",
|
|
"evidence": "Yes, FL110 is listed at $385, and it's the cheapest nonstop that departs before noon."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "booking-flight",
|
|
"condition": "strawman",
|
|
"wall_time": 181.1,
|
|
"error": null,
|
|
"n_user_turns": 4,
|
|
"n_slow_steps": 21,
|
|
"n_to_slow": 0,
|
|
"n_ask_user": 4,
|
|
"voice_latency_p50": 15.665,
|
|
"voice_latency_p90": 20.544,
|
|
"voice_latency_mean": 17.052,
|
|
"n_latency_samples": 5,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"flight": {
|
|
"airline": "Cirrus",
|
|
"arrive": "13:05",
|
|
"date": "2026-07-21",
|
|
"depart": "09:45",
|
|
"from": "SFO",
|
|
"id": "FL110",
|
|
"price": 385,
|
|
"stops": 0,
|
|
"to": "JFK"
|
|
},
|
|
"passenger": "Daniel Ortiz",
|
|
"checks": {
|
|
"date": true,
|
|
"depart": true,
|
|
"stops": true,
|
|
"price": true,
|
|
"passenger": true
|
|
}
|
|
},
|
|
"probe_correct": 0.5,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "wrong",
|
|
"evidence": "Agent only discussed morning nonstops under $400, saying FL110 is 'the only nonstop that leaves before noon and stays under $400' and mentioned FL108. Did not convey there are 4 nonstop options total."
|
|
},
|
|
{
|
|
"q": 1,
|
|
"grade": "correct",
|
|
"evidence": "Agent stated FL110 for $385 before booking."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "webmail-report",
|
|
"condition": "duplex",
|
|
"wall_time": 169.7,
|
|
"error": null,
|
|
"n_user_turns": 4,
|
|
"n_slow_steps": 19,
|
|
"n_to_slow": 4,
|
|
"n_ask_user": 3,
|
|
"voice_latency_p50": 2.67,
|
|
"voice_latency_p90": 2.837,
|
|
"voice_latency_mean": 2.923,
|
|
"n_latency_samples": 4,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"email": {
|
|
"attachment": "Q3_financial_report_final.pdf",
|
|
"body": "Hi,\r\n\r\nPlease find attached the Q3 financial report (final).\r\n\r\nThanks.",
|
|
"subject": "Q3 Report",
|
|
"to": "maria.lopez@acctfirm.com"
|
|
},
|
|
"checks": {
|
|
"to": true,
|
|
"attachment": true,
|
|
"subject": true
|
|
}
|
|
},
|
|
"probe_correct": 1.0,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "correct",
|
|
"evidence": "I'm attaching the Q3_financial_report_final.pdf\u2014that's the final version, not the draft."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "webmail-report",
|
|
"condition": "duplex",
|
|
"wall_time": 172.1,
|
|
"error": null,
|
|
"n_user_turns": 6,
|
|
"n_slow_steps": 18,
|
|
"n_to_slow": 5,
|
|
"n_ask_user": 3,
|
|
"voice_latency_p50": 1.675,
|
|
"voice_latency_p90": 2.533,
|
|
"voice_latency_mean": 1.882,
|
|
"n_latency_samples": 6,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"email": {
|
|
"attachment": "Q3_financial_report_final.pdf",
|
|
"body": "Hi Maria,\r\n\r\nPlease find attached the final Q3 financial report.\r\n\r\nBest regards",
|
|
"subject": "Q3 Report",
|
|
"to": "maria.lopez@acctfirm.com"
|
|
},
|
|
"checks": {
|
|
"to": true,
|
|
"attachment": true,
|
|
"subject": true
|
|
}
|
|
},
|
|
"probe_correct": 1.0,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "correct",
|
|
"evidence": "The file I'm attaching is Q3_financial_report_final.pdf \u2014 so yes, that's the final version, not the draft."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "webmail-report",
|
|
"condition": "strawman",
|
|
"wall_time": 143.8,
|
|
"error": null,
|
|
"n_user_turns": 4,
|
|
"n_slow_steps": 19,
|
|
"n_to_slow": 0,
|
|
"n_ask_user": 3,
|
|
"voice_latency_p50": 19.451,
|
|
"voice_latency_p90": 19.75,
|
|
"voice_latency_mean": 15.048,
|
|
"n_latency_samples": 5,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"email": {
|
|
"attachment": "Q3_financial_report_final.pdf",
|
|
"body": "Hi Maria,\r\n\r\nPlease find attached the final Q3 financial report.\r\n\r\nThank you.",
|
|
"subject": "Q3 Financial Report (Final)",
|
|
"to": "maria.lopez@acctfirm.com"
|
|
},
|
|
"checks": {
|
|
"to": true,
|
|
"attachment": true,
|
|
"subject": true
|
|
}
|
|
},
|
|
"probe_correct": 1.0,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "correct",
|
|
"evidence": "Confirmed \u2014 the attachment is Q3_financial_report_final.pdf, the final version, not the draft."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "webmail-report",
|
|
"condition": "strawman",
|
|
"wall_time": 138.0,
|
|
"error": null,
|
|
"n_user_turns": 3,
|
|
"n_slow_steps": 17,
|
|
"n_to_slow": 0,
|
|
"n_ask_user": 3,
|
|
"voice_latency_p50": 17.49,
|
|
"voice_latency_p90": 18.852,
|
|
"voice_latency_mean": 16.402,
|
|
"n_latency_samples": 4,
|
|
"success": true,
|
|
"partial": 1.0,
|
|
"details": {
|
|
"email": {
|
|
"attachment": "Q3_financial_report_final.pdf",
|
|
"body": "Hi Maria,\r\n\r\nPlease find attached the final Q3 financial report. Let me know if you have any questions.\r\n\r\nBest regards",
|
|
"subject": "Q3 Report",
|
|
"to": "maria.lopez@acctfirm.com"
|
|
},
|
|
"checks": {
|
|
"to": true,
|
|
"attachment": true,
|
|
"subject": true
|
|
}
|
|
},
|
|
"probe_correct": 1.0,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "correct",
|
|
"evidence": "with Q3_financial_report_final.pdf attached... the final Q3 financial report PDF attached"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "meeting-helper",
|
|
"condition": "duplex",
|
|
"wall_time": 130.1,
|
|
"error": null,
|
|
"n_user_turns": 4,
|
|
"n_slow_steps": 12,
|
|
"n_to_slow": 0,
|
|
"n_ask_user": 0,
|
|
"voice_latency_p50": 2.405,
|
|
"voice_latency_p90": 2.428,
|
|
"voice_latency_mean": 2.337,
|
|
"n_latency_samples": 4,
|
|
"success": null,
|
|
"partial": null,
|
|
"details": "judged via probe answers",
|
|
"probe_correct": 1.0,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "correct",
|
|
"evidence": "Atlas is confirmed for September 18th"
|
|
},
|
|
{
|
|
"q": 1,
|
|
"grade": "correct",
|
|
"evidence": "drop-off improved from thirty-four percent to nineteen percent"
|
|
},
|
|
{
|
|
"q": 2,
|
|
"grade": "correct",
|
|
"evidence": "Marcus is owning the CDN renewal...August 15th deadline before the August 30th contract expiration"
|
|
},
|
|
{
|
|
"q": 3,
|
|
"grade": "correct",
|
|
"evidence": "Marketing's getting an extra one-fifty K for the launch campaign"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "meeting-helper",
|
|
"condition": "duplex",
|
|
"wall_time": 148.2,
|
|
"error": null,
|
|
"n_user_turns": 4,
|
|
"n_slow_steps": 12,
|
|
"n_to_slow": 0,
|
|
"n_ask_user": 0,
|
|
"voice_latency_p50": 2.062,
|
|
"voice_latency_p90": 2.217,
|
|
"voice_latency_mean": 2.018,
|
|
"n_latency_samples": 4,
|
|
"success": null,
|
|
"partial": null,
|
|
"details": "judged via probe answers",
|
|
"probe_correct": 1.0,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "correct",
|
|
"evidence": "Atlas launch is confirmed for September eighteenth"
|
|
},
|
|
{
|
|
"q": 1,
|
|
"grade": "correct",
|
|
"evidence": "improved the drop-off rate from thirty-four percent down to nineteen percent"
|
|
},
|
|
{
|
|
"q": 2,
|
|
"grade": "correct",
|
|
"evidence": "Marcus has taken ownership of that with an August 15th deadline"
|
|
},
|
|
{
|
|
"q": 3,
|
|
"grade": "correct",
|
|
"evidence": "Marketing's also getting an extra one hundred fifty thousand dollars"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "meeting-helper",
|
|
"condition": "strawman",
|
|
"wall_time": 123.8,
|
|
"error": null,
|
|
"n_user_turns": 3,
|
|
"n_slow_steps": 14,
|
|
"n_to_slow": 0,
|
|
"n_ask_user": 0,
|
|
"voice_latency_p50": 12.009,
|
|
"voice_latency_p90": 12.009,
|
|
"voice_latency_mean": 8.15,
|
|
"n_latency_samples": 3,
|
|
"success": null,
|
|
"partial": null,
|
|
"details": "judged via probe answers",
|
|
"probe_correct": 1.0,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "correct",
|
|
"evidence": "Atlas launch is confirmed for September 18th"
|
|
},
|
|
{
|
|
"q": 1,
|
|
"grade": "correct",
|
|
"evidence": "onboarding flow cut drop-off from 34% down to 19%"
|
|
},
|
|
{
|
|
"q": 2,
|
|
"grade": "correct",
|
|
"evidence": "Marcus owns the CDN renewal with a deadline of August 15, contract expires August 30"
|
|
},
|
|
{
|
|
"q": 3,
|
|
"grade": "correct",
|
|
"evidence": "marketing gets an additional $150,000 for the Atlas launch campaign"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"task": "meeting-helper",
|
|
"condition": "strawman",
|
|
"wall_time": 128.7,
|
|
"error": null,
|
|
"n_user_turns": 3,
|
|
"n_slow_steps": 14,
|
|
"n_to_slow": 0,
|
|
"n_ask_user": 0,
|
|
"voice_latency_p50": 10.903,
|
|
"voice_latency_p90": 10.903,
|
|
"voice_latency_mean": 7.769,
|
|
"n_latency_samples": 3,
|
|
"success": null,
|
|
"partial": null,
|
|
"details": "judged via probe answers",
|
|
"probe_correct": 1.0,
|
|
"probe_grades": [
|
|
{
|
|
"q": 0,
|
|
"grade": "correct",
|
|
"evidence": "Atlas launch is confirmed for September 18th"
|
|
},
|
|
{
|
|
"q": 1,
|
|
"grade": "correct",
|
|
"evidence": "cut drop-off from 34% to 19% in the A/B test"
|
|
},
|
|
{
|
|
"q": 2,
|
|
"grade": "correct",
|
|
"evidence": "Marcus owns the CDN renewal, due August 15; contract expires August 30"
|
|
},
|
|
{
|
|
"q": 3,
|
|
"grade": "correct",
|
|
"evidence": "marketing gets an extra $150,000 for the Atlas launch campaign"
|
|
}
|
|
]
|
|
}
|
|
] |