[
 {
  "task": "forms-insurance",
  "condition": "duplex",
  "wall_time": 274.9,
  "error": null,
  "n_user_turns": 9,
  "n_slow_steps": 30,
  "n_to_slow": 10,
  "n_ask_user": 4,
  "voice_latency_p50": 2.451,
  "voice_latency_p90": 2.854,
  "voice_latency_mean": 2.587,
  "n_latency_samples": 9,
  "success": true,
  "partial": 1.0,
  "details": {
   "full_name": "Sarah Mitchell",
   "dob": "1989-04-12",
   "email": "sarah.mitchell89@gmail.com",
   "phone": "415-555-0173",
   "street": "2847 Fillmore Street",
   "city": "San Francisco",
   "zip": "94123",
   "vehicle_make": "Toyota",
   "vehicle_model": "RAV4",
   "vehicle_year": "2021",
   "coverage": "full",
   "prior_claims": "no"
  },
  "probe_correct": null
 },
 {
  "task": "forms-insurance",
  "condition": "duplex",
  "wall_time": 321.8,
  "error": null,
  "n_user_turns": 9,
  "n_slow_steps": 33,
  "n_to_slow": 8,
  "n_ask_user": 5,
  "voice_latency_p50": 2.39,
  "voice_latency_p90": 2.863,
  "voice_latency_mean": 2.53,
  "n_latency_samples": 9,
  "success": true,
  "partial": 1.0,
  "details": {
   "full_name": "Sarah Mitchell",
   "dob": "1989-04-12",
   "email": "sarah.mitchell89@gmail.com",
   "phone": "415-555-0173",
   "street": "2847 Fillmore Street",
   "city": "San Francisco",
   "zip": "94123",
   "vehicle_make": "Toyota",
   "vehicle_model": "RAV4",
   "vehicle_year": "2021",
   "coverage": "full",
   "prior_claims": "no"
  },
  "probe_correct": null
 },
 {
  "task": "forms-insurance",
  "condition": "strawman",
  "wall_time": 225.9,
  "error": null,
  "n_user_turns": 6,
  "n_slow_steps": 31,
  "n_to_slow": 0,
  "n_ask_user": 6,
  "voice_latency_p50": 18.296,
  "voice_latency_p90": 22.962,
  "voice_latency_mean": 18.505,
  "n_latency_samples": 7,
  "success": true,
  "partial": 1.0,
  "details": {
   "full_name": "Sarah Mitchell",
   "dob": "1989-04-12",
   "email": "sarah.mitchell89@gmail.com",
   "phone": "415-555-0173",
   "street": "2847 Fillmore Street",
   "city": "San Francisco",
   "zip": "94123",
   "vehicle_make": "Toyota",
   "vehicle_model": "RAV4",
   "vehicle_year": "2021",
   "coverage": "full",
   "prior_claims": "no"
  },
  "probe_correct": null
 },
 {
  "task": "forms-insurance",
  "condition": "strawman",
  "wall_time": 247.3,
  "error": null,
  "n_user_turns": 10,
  "n_slow_steps": 36,
  "n_to_slow": 0,
  "n_ask_user": 7,
  "voice_latency_p50": 10.862,
  "voice_latency_p90": 12.52,
  "voice_latency_mean": 10.134,
  "n_latency_samples": 11,
  "success": true,
  "partial": 1.0,
  "details": {
   "full_name": "Sarah Mitchell",
   "dob": "1989-04-12",
   "email": "sarah.mitchell89@gmail.com",
   "phone": "415-555-0173",
   "street": "2847 Fillmore Street",
   "city": "San Francisco",
   "zip": "94123",
   "vehicle_make": "Toyota",
   "vehicle_model": "RAV4",
   "vehicle_year": "2021",
   "coverage": "full",
   "prior_claims": "no"
  },
  "probe_correct": null
 },
 {
  "task": "booking-flight",
  "condition": "duplex",
  "wall_time": 239.0,
  "error": null,
  "n_user_turns": 6,
  "n_slow_steps": 22,
  "n_to_slow": 6,
  "n_ask_user": 3,
  "voice_latency_p50": 1.847,
  "voice_latency_p90": 2.039,
  "voice_latency_mean": 1.857,
  "n_latency_samples": 6,
  "success": true,
  "partial": 1.0,
  "details": {
   "flight": {
    "airline": "Cirrus",
    "arrive": "13:05",
    "date": "2026-07-21",
    "depart": "09:45",
    "from": "SFO",
    "id": "FL110",
    "price": 385,
    "stops": 0,
    "to": "JFK"
   },
   "passenger": "Daniel Ortiz",
   "checks": {
    "date": true,
    "depart": true,
    "stops": true,
    "price": true,
    "passenger": true
   }
  },
  "probe_correct": 0.5,
  "probe_grades": [
   {
    "q": 0,
    "grade": "wrong",
    "evidence": "Agent said only two nonstop morning flights (FL110 $385, Delta $420); gold says there are 4 nonstop options total. Agent incorrectly claimed those are the only nonstop flights."
   },
   {
    "q": 1,
    "grade": "correct",
    "evidence": "'the total is $385 for your flight'"
   }
  ]
 },
 {
  "task": "booking-flight",
  "condition": "duplex",
  "wall_time": 201.8,
  "error": null,
  "n_user_turns": 5,
  "n_slow_steps": 20,
  "n_to_slow": 6,
  "n_ask_user": 3,
  "voice_latency_p50": 1.605,
  "voice_latency_p90": 1.887,
  "voice_latency_mean": 1.71,
  "n_latency_samples": 5,
  "success": true,
  "partial": 1.0,
  "details": {
   "flight": {
    "airline": "Cirrus",
    "arrive": "13:05",
    "date": "2026-07-21",
    "depart": "09:45",
    "from": "SFO",
    "id": "FL110",
    "price": 385,
    "stops": 0,
    "to": "JFK"
   },
   "passenger": "Daniel Ortiz",
   "checks": {
    "date": true,
    "depart": true,
    "stops": true,
    "price": true,
    "passenger": true
   }
  },
  "probe_correct": 0.5,
  "probe_grades": [
   {
    "q": 0,
    "grade": "wrong",
    "evidence": "Agent said 'three options' for nonstop morning flights under $400, but the gold says 4 nonstop options total (06:15 $420, 09:45 $385, 13:00 $455, 17:25 $340). Agent missed the 17:25 $340 option and framed the count incorrectly."
   },
   {
    "q": 1,
    "grade": "correct",
    "evidence": "'yes, three eighty five is the final price' - matches gold $385."
   }
  ]
 },
 {
  "task": "booking-flight",
  "condition": "strawman",
  "wall_time": 235.6,
  "error": null,
  "n_user_turns": 5,
  "n_slow_steps": 25,
  "n_to_slow": 0,
  "n_ask_user": 3,
  "voice_latency_p50": 19.739,
  "voice_latency_p90": 21.142,
  "voice_latency_mean": 18.185,
  "n_latency_samples": 6,
  "success": true,
  "partial": 1.0,
  "details": {
   "flight": {
    "airline": "Cirrus",
    "arrive": "13:05",
    "date": "2026-07-21",
    "depart": "09:45",
    "from": "SFO",
    "id": "FL110",
    "price": 385,
    "stops": 0,
    "to": "JFK"
   },
   "passenger": "Daniel Ortiz",
   "checks": {
    "date": true,
    "depart": true,
    "stops": true,
    "price": true,
    "passenger": true
   }
  },
  "probe_correct": 1.0,
  "probe_grades": [
   {
    "q": 0,
    "grade": "correct",
    "evidence": "There are four nonstops: FL108 Delta at 06:15 for $420, FL110 Cirrus at 09:45 for $385, FL112 Blue Sky at 13:00 for $455, and FL114 Acme at 17:25 for $340."
   },
   {
    "q": 1,
    "grade": "correct",
    "evidence": "Yes, FL110 is listed at $385, and it's the cheapest nonstop that departs before noon."
   }
  ]
 },
 {
  "task": "booking-flight",
  "condition": "strawman",
  "wall_time": 181.1,
  "error": null,
  "n_user_turns": 4,
  "n_slow_steps": 21,
  "n_to_slow": 0,
  "n_ask_user": 4,
  "voice_latency_p50": 15.665,
  "voice_latency_p90": 20.544,
  "voice_latency_mean": 17.052,
  "n_latency_samples": 5,
  "success": true,
  "partial": 1.0,
  "details": {
   "flight": {
    "airline": "Cirrus",
    "arrive": "13:05",
    "date": "2026-07-21",
    "depart": "09:45",
    "from": "SFO",
    "id": "FL110",
    "price": 385,
    "stops": 0,
    "to": "JFK"
   },
   "passenger": "Daniel Ortiz",
   "checks": {
    "date": true,
    "depart": true,
    "stops": true,
    "price": true,
    "passenger": true
   }
  },
  "probe_correct": 0.5,
  "probe_grades": [
   {
    "q": 0,
    "grade": "wrong",
    "evidence": "Agent only discussed morning nonstops under $400, saying FL110 is 'the only nonstop that leaves before noon and stays under $400' and mentioned FL108. Did not convey there are 4 nonstop options total."
   },
   {
    "q": 1,
    "grade": "correct",
    "evidence": "Agent stated FL110 for $385 before booking."
   }
  ]
 },
 {
  "task": "webmail-report",
  "condition": "duplex",
  "wall_time": 169.7,
  "error": null,
  "n_user_turns": 4,
  "n_slow_steps": 19,
  "n_to_slow": 4,
  "n_ask_user": 3,
  "voice_latency_p50": 2.67,
  "voice_latency_p90": 2.837,
  "voice_latency_mean": 2.923,
  "n_latency_samples": 4,
  "success": true,
  "partial": 1.0,
  "details": {
   "email": {
    "attachment": "Q3_financial_report_final.pdf",
    "body": "Hi,\r\n\r\nPlease find attached the Q3 financial report (final).\r\n\r\nThanks.",
    "subject": "Q3 Report",
    "to": "maria.lopez@acctfirm.com"
   },
   "checks": {
    "to": true,
    "attachment": true,
    "subject": true
   }
  },
  "probe_correct": 1.0,
  "probe_grades": [
   {
    "q": 0,
    "grade": "correct",
    "evidence": "I'm attaching the Q3_financial_report_final.pdf\u2014that's the final version, not the draft."
   }
  ]
 },
 {
  "task": "webmail-report",
  "condition": "duplex",
  "wall_time": 172.1,
  "error": null,
  "n_user_turns": 6,
  "n_slow_steps": 18,
  "n_to_slow": 5,
  "n_ask_user": 3,
  "voice_latency_p50": 1.675,
  "voice_latency_p90": 2.533,
  "voice_latency_mean": 1.882,
  "n_latency_samples": 6,
  "success": true,
  "partial": 1.0,
  "details": {
   "email": {
    "attachment": "Q3_financial_report_final.pdf",
    "body": "Hi Maria,\r\n\r\nPlease find attached the final Q3 financial report.\r\n\r\nBest regards",
    "subject": "Q3 Report",
    "to": "maria.lopez@acctfirm.com"
   },
   "checks": {
    "to": true,
    "attachment": true,
    "subject": true
   }
  },
  "probe_correct": 1.0,
  "probe_grades": [
   {
    "q": 0,
    "grade": "correct",
    "evidence": "The file I'm attaching is Q3_financial_report_final.pdf \u2014 so yes, that's the final version, not the draft."
   }
  ]
 },
 {
  "task": "webmail-report",
  "condition": "strawman",
  "wall_time": 143.8,
  "error": null,
  "n_user_turns": 4,
  "n_slow_steps": 19,
  "n_to_slow": 0,
  "n_ask_user": 3,
  "voice_latency_p50": 19.451,
  "voice_latency_p90": 19.75,
  "voice_latency_mean": 15.048,
  "n_latency_samples": 5,
  "success": true,
  "partial": 1.0,
  "details": {
   "email": {
    "attachment": "Q3_financial_report_final.pdf",
    "body": "Hi Maria,\r\n\r\nPlease find attached the final Q3 financial report.\r\n\r\nThank you.",
    "subject": "Q3 Financial Report (Final)",
    "to": "maria.lopez@acctfirm.com"
   },
   "checks": {
    "to": true,
    "attachment": true,
    "subject": true
   }
  },
  "probe_correct": 1.0,
  "probe_grades": [
   {
    "q": 0,
    "grade": "correct",
    "evidence": "Confirmed \u2014 the attachment is Q3_financial_report_final.pdf, the final version, not the draft."
   }
  ]
 },
 {
  "task": "webmail-report",
  "condition": "strawman",
  "wall_time": 138.0,
  "error": null,
  "n_user_turns": 3,
  "n_slow_steps": 17,
  "n_to_slow": 0,
  "n_ask_user": 3,
  "voice_latency_p50": 17.49,
  "voice_latency_p90": 18.852,
  "voice_latency_mean": 16.402,
  "n_latency_samples": 4,
  "success": true,
  "partial": 1.0,
  "details": {
   "email": {
    "attachment": "Q3_financial_report_final.pdf",
    "body": "Hi Maria,\r\n\r\nPlease find attached the final Q3 financial report. Let me know if you have any questions.\r\n\r\nBest regards",
    "subject": "Q3 Report",
    "to": "maria.lopez@acctfirm.com"
   },
   "checks": {
    "to": true,
    "attachment": true,
    "subject": true
   }
  },
  "probe_correct": 1.0,
  "probe_grades": [
   {
    "q": 0,
    "grade": "correct",
    "evidence": "with Q3_financial_report_final.pdf attached... the final Q3 financial report PDF attached"
   }
  ]
 },
 {
  "task": "meeting-helper",
  "condition": "duplex",
  "wall_time": 130.1,
  "error": null,
  "n_user_turns": 4,
  "n_slow_steps": 12,
  "n_to_slow": 0,
  "n_ask_user": 0,
  "voice_latency_p50": 2.405,
  "voice_latency_p90": 2.428,
  "voice_latency_mean": 2.337,
  "n_latency_samples": 4,
  "success": null,
  "partial": null,
  "details": "judged via probe answers",
  "probe_correct": 1.0,
  "probe_grades": [
   {
    "q": 0,
    "grade": "correct",
    "evidence": "Atlas is confirmed for September 18th"
   },
   {
    "q": 1,
    "grade": "correct",
    "evidence": "drop-off improved from thirty-four percent to nineteen percent"
   },
   {
    "q": 2,
    "grade": "correct",
    "evidence": "Marcus is owning the CDN renewal...August 15th deadline before the August 30th contract expiration"
   },
   {
    "q": 3,
    "grade": "correct",
    "evidence": "Marketing's getting an extra one-fifty K for the launch campaign"
   }
  ]
 },
 {
  "task": "meeting-helper",
  "condition": "duplex",
  "wall_time": 148.2,
  "error": null,
  "n_user_turns": 4,
  "n_slow_steps": 12,
  "n_to_slow": 0,
  "n_ask_user": 0,
  "voice_latency_p50": 2.062,
  "voice_latency_p90": 2.217,
  "voice_latency_mean": 2.018,
  "n_latency_samples": 4,
  "success": null,
  "partial": null,
  "details": "judged via probe answers",
  "probe_correct": 1.0,
  "probe_grades": [
   {
    "q": 0,
    "grade": "correct",
    "evidence": "Atlas launch is confirmed for September eighteenth"
   },
   {
    "q": 1,
    "grade": "correct",
    "evidence": "improved the drop-off rate from thirty-four percent down to nineteen percent"
   },
   {
    "q": 2,
    "grade": "correct",
    "evidence": "Marcus has taken ownership of that with an August 15th deadline"
   },
   {
    "q": 3,
    "grade": "correct",
    "evidence": "Marketing's also getting an extra one hundred fifty thousand dollars"
   }
  ]
 },
 {
  "task": "meeting-helper",
  "condition": "strawman",
  "wall_time": 123.8,
  "error": null,
  "n_user_turns": 3,
  "n_slow_steps": 14,
  "n_to_slow": 0,
  "n_ask_user": 0,
  "voice_latency_p50": 12.009,
  "voice_latency_p90": 12.009,
  "voice_latency_mean": 8.15,
  "n_latency_samples": 3,
  "success": null,
  "partial": null,
  "details": "judged via probe answers",
  "probe_correct": 1.0,
  "probe_grades": [
   {
    "q": 0,
    "grade": "correct",
    "evidence": "Atlas launch is confirmed for September 18th"
   },
   {
    "q": 1,
    "grade": "correct",
    "evidence": "onboarding flow cut drop-off from 34% down to 19%"
   },
   {
    "q": 2,
    "grade": "correct",
    "evidence": "Marcus owns the CDN renewal with a deadline of August 15, contract expires August 30"
   },
   {
    "q": 3,
    "grade": "correct",
    "evidence": "marketing gets an additional $150,000 for the Atlas launch campaign"
   }
  ]
 },
 {
  "task": "meeting-helper",
  "condition": "strawman",
  "wall_time": 128.7,
  "error": null,
  "n_user_turns": 3,
  "n_slow_steps": 14,
  "n_to_slow": 0,
  "n_ask_user": 0,
  "voice_latency_p50": 10.903,
  "voice_latency_p90": 10.903,
  "voice_latency_mean": 7.769,
  "n_latency_samples": 3,
  "success": null,
  "partial": null,
  "details": "judged via probe answers",
  "probe_correct": 1.0,
  "probe_grades": [
   {
    "q": 0,
    "grade": "correct",
    "evidence": "Atlas launch is confirmed for September 18th"
   },
   {
    "q": 1,
    "grade": "correct",
    "evidence": "cut drop-off from 34% to 19% in the A/B test"
   },
   {
    "q": 2,
    "grade": "correct",
    "evidence": "Marcus owns the CDN renewal, due August 15; contract expires August 30"
   },
   {
    "q": 3,
    "grade": "correct",
    "evidence": "marketing gets an extra $150,000 for the Atlas launch campaign"
   }
  ]
 }
]