liqiang
b119135836
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
530 lines
16 KiB
JSON
530 lines
16 KiB
JSON
{
|
|
"results": [
|
|
{
|
|
"test": {
|
|
"test_id": "TC-R1-HTTP-RF-001-T2",
|
|
"trajectory_id": "HTTP-RF-001",
|
|
"focus_turn": 2,
|
|
"description": "Refund flow must include verify_refund_eligibility before process_refund (R1)",
|
|
"assertion": {
|
|
"type": "step_present",
|
|
"params": {
|
|
"tool": "verify_refund_eligibility"
|
|
}
|
|
}
|
|
},
|
|
"buggy": {
|
|
"passed": false,
|
|
"detail": "verify_refund_eligibility calls=0",
|
|
"trajectory": {
|
|
"trajectory_id": "HTTP-RF-001::buggy",
|
|
"source_trajectory_id": "HTTP-RF-001",
|
|
"implementation": "buggy",
|
|
"task_input": {
|
|
"intent": "refund",
|
|
"order_id": "ORD-58-A"
|
|
},
|
|
"final_status": "success",
|
|
"turns": [
|
|
{
|
|
"index": 0,
|
|
"role": "user",
|
|
"content": "{\"intent\": \"refund\", \"order_id\": \"ORD-58-A\"}"
|
|
},
|
|
{
|
|
"index": 1,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "query_order",
|
|
"method": "GET",
|
|
"path": "/orders/ORD-58-A",
|
|
"request": null,
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 1.833,
|
|
"response": {
|
|
"order_id": "ORD-58-A",
|
|
"status": "paid",
|
|
"sku": "SKU-42"
|
|
}
|
|
},
|
|
{
|
|
"index": 2,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "process_refund",
|
|
"method": "POST",
|
|
"path": "/refund/process",
|
|
"request": {
|
|
"order_id": "ORD-58-A"
|
|
},
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.833,
|
|
"response": {
|
|
"order_id": "ORD-58-A",
|
|
"refund_id": "RF-LIVE-1"
|
|
}
|
|
},
|
|
{
|
|
"index": 3,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "notify_user",
|
|
"method": "POST",
|
|
"path": "/notifications",
|
|
"request": {
|
|
"status": "success"
|
|
},
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.678,
|
|
"response": {
|
|
"sent": true,
|
|
"status": "success"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"fixed": {
|
|
"passed": true,
|
|
"detail": "verify_refund_eligibility calls=1",
|
|
"trajectory": {
|
|
"trajectory_id": "HTTP-RF-001::fixed",
|
|
"source_trajectory_id": "HTTP-RF-001",
|
|
"implementation": "fixed",
|
|
"task_input": {
|
|
"intent": "refund",
|
|
"order_id": "ORD-58-A"
|
|
},
|
|
"final_status": "success",
|
|
"turns": [
|
|
{
|
|
"index": 0,
|
|
"role": "user",
|
|
"content": "{\"intent\": \"refund\", \"order_id\": \"ORD-58-A\"}"
|
|
},
|
|
{
|
|
"index": 1,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "query_order",
|
|
"method": "GET",
|
|
"path": "/orders/ORD-58-A",
|
|
"request": null,
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.498,
|
|
"response": {
|
|
"order_id": "ORD-58-A",
|
|
"status": "paid",
|
|
"sku": "SKU-42"
|
|
}
|
|
},
|
|
{
|
|
"index": 2,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "verify_refund_eligibility",
|
|
"method": "POST",
|
|
"path": "/refund/eligibility",
|
|
"request": {
|
|
"order_id": "ORD-58-A"
|
|
},
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.379,
|
|
"response": {
|
|
"order_id": "ORD-58-A",
|
|
"eligible": true
|
|
}
|
|
},
|
|
{
|
|
"index": 3,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "process_refund",
|
|
"method": "POST",
|
|
"path": "/refund/process",
|
|
"request": {
|
|
"order_id": "ORD-58-A"
|
|
},
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.37,
|
|
"response": {
|
|
"order_id": "ORD-58-A",
|
|
"refund_id": "RF-LIVE-1"
|
|
}
|
|
},
|
|
{
|
|
"index": 4,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "notify_user",
|
|
"method": "POST",
|
|
"path": "/notifications",
|
|
"request": {
|
|
"status": "success"
|
|
},
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.315,
|
|
"response": {
|
|
"sent": true,
|
|
"status": "success"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"test": {
|
|
"test_id": "TC-R2-LATENCY-HTTP-INV-001-T2",
|
|
"trajectory_id": "HTTP-INV-001",
|
|
"focus_turn": 2,
|
|
"description": "check_stock must complete within 250 ms (R2)",
|
|
"assertion": {
|
|
"type": "latency_under",
|
|
"params": {
|
|
"tool": "check_stock",
|
|
"threshold_ms": 250
|
|
}
|
|
}
|
|
},
|
|
"buggy": {
|
|
"passed": false,
|
|
"detail": "check_stock max_latency_ms=350.383, threshold_ms=250.000",
|
|
"trajectory": {
|
|
"trajectory_id": "HTTP-INV-001::buggy",
|
|
"source_trajectory_id": "HTTP-INV-001",
|
|
"implementation": "buggy",
|
|
"task_input": {
|
|
"intent": "order_status",
|
|
"order_id": "ORD-58-B",
|
|
"sku": "SKU-42"
|
|
},
|
|
"final_status": "failed",
|
|
"turns": [
|
|
{
|
|
"index": 0,
|
|
"role": "user",
|
|
"content": "{\"intent\": \"order_status\", \"order_id\": \"ORD-58-B\", \"sku\": \"SKU-42\"}"
|
|
},
|
|
{
|
|
"index": 1,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "query_order",
|
|
"method": "GET",
|
|
"path": "/orders/ORD-58-B",
|
|
"request": null,
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.308,
|
|
"response": {
|
|
"order_id": "ORD-58-B",
|
|
"status": "paid",
|
|
"sku": "SKU-42"
|
|
}
|
|
},
|
|
{
|
|
"index": 2,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "check_stock",
|
|
"method": "GET",
|
|
"path": "/inventory/SKU-42",
|
|
"request": null,
|
|
"http_status": null,
|
|
"status": "error",
|
|
"latency_ms": 350.383,
|
|
"error": "TimeoutError: timed out"
|
|
},
|
|
{
|
|
"index": 3,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "notify_user",
|
|
"method": "POST",
|
|
"path": "/notifications",
|
|
"request": {
|
|
"status": "failed"
|
|
},
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 1.481,
|
|
"response": {
|
|
"sent": true,
|
|
"status": "failed"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"fixed": {
|
|
"passed": true,
|
|
"detail": "check_stock max_latency_ms=101.234, threshold_ms=250.000",
|
|
"trajectory": {
|
|
"trajectory_id": "HTTP-INV-001::fixed",
|
|
"source_trajectory_id": "HTTP-INV-001",
|
|
"implementation": "fixed",
|
|
"task_input": {
|
|
"intent": "order_status",
|
|
"order_id": "ORD-58-B",
|
|
"sku": "SKU-42"
|
|
},
|
|
"final_status": "success",
|
|
"turns": [
|
|
{
|
|
"index": 0,
|
|
"role": "user",
|
|
"content": "{\"intent\": \"order_status\", \"order_id\": \"ORD-58-B\", \"sku\": \"SKU-42\"}"
|
|
},
|
|
{
|
|
"index": 1,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "query_order",
|
|
"method": "GET",
|
|
"path": "/orders/ORD-58-B",
|
|
"request": null,
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.843,
|
|
"response": {
|
|
"order_id": "ORD-58-B",
|
|
"status": "paid",
|
|
"sku": "SKU-42"
|
|
}
|
|
},
|
|
{
|
|
"index": 2,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "check_stock",
|
|
"method": "GET",
|
|
"path": "/inventory/SKU-42",
|
|
"request": null,
|
|
"http_status": null,
|
|
"status": "error",
|
|
"latency_ms": 101.234,
|
|
"error": "TimeoutError: timed out"
|
|
},
|
|
{
|
|
"index": 3,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "check_stock",
|
|
"method": "GET",
|
|
"path": "/inventory/SKU-42?degraded=1",
|
|
"request": null,
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 1.289,
|
|
"response": {
|
|
"sku": "SKU-42",
|
|
"stock": 12,
|
|
"source": "cache",
|
|
"degraded": true
|
|
}
|
|
},
|
|
{
|
|
"index": 4,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "notify_user",
|
|
"method": "POST",
|
|
"path": "/notifications",
|
|
"request": {
|
|
"status": "success"
|
|
},
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.78,
|
|
"response": {
|
|
"sent": true,
|
|
"status": "success"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"test": {
|
|
"test_id": "TC-R2-STATUS-HTTP-INV-001-T2",
|
|
"trajectory_id": "HTTP-INV-001",
|
|
"focus_turn": 2,
|
|
"description": "check_stock origin timeout must use degraded cache route, resulting in successful flow (R2)",
|
|
"assertion": {
|
|
"type": "final_status_is",
|
|
"params": {
|
|
"value": "success"
|
|
}
|
|
}
|
|
},
|
|
"buggy": {
|
|
"passed": false,
|
|
"detail": "final_status=failed, expected=success",
|
|
"trajectory": {
|
|
"trajectory_id": "HTTP-INV-001::buggy",
|
|
"source_trajectory_id": "HTTP-INV-001",
|
|
"implementation": "buggy",
|
|
"task_input": {
|
|
"intent": "order_status",
|
|
"order_id": "ORD-58-B",
|
|
"sku": "SKU-42"
|
|
},
|
|
"final_status": "failed",
|
|
"turns": [
|
|
{
|
|
"index": 0,
|
|
"role": "user",
|
|
"content": "{\"intent\": \"order_status\", \"order_id\": \"ORD-58-B\", \"sku\": \"SKU-42\"}"
|
|
},
|
|
{
|
|
"index": 1,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "query_order",
|
|
"method": "GET",
|
|
"path": "/orders/ORD-58-B",
|
|
"request": null,
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.608,
|
|
"response": {
|
|
"order_id": "ORD-58-B",
|
|
"status": "paid",
|
|
"sku": "SKU-42"
|
|
}
|
|
},
|
|
{
|
|
"index": 2,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "check_stock",
|
|
"method": "GET",
|
|
"path": "/inventory/SKU-42",
|
|
"request": null,
|
|
"http_status": null,
|
|
"status": "error",
|
|
"latency_ms": 355.612,
|
|
"error": "TimeoutError: timed out"
|
|
},
|
|
{
|
|
"index": 3,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "notify_user",
|
|
"method": "POST",
|
|
"path": "/notifications",
|
|
"request": {
|
|
"status": "failed"
|
|
},
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 1.19,
|
|
"response": {
|
|
"sent": true,
|
|
"status": "failed"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"fixed": {
|
|
"passed": true,
|
|
"detail": "final_status=success, expected=success",
|
|
"trajectory": {
|
|
"trajectory_id": "HTTP-INV-001::fixed",
|
|
"source_trajectory_id": "HTTP-INV-001",
|
|
"implementation": "fixed",
|
|
"task_input": {
|
|
"intent": "order_status",
|
|
"order_id": "ORD-58-B",
|
|
"sku": "SKU-42"
|
|
},
|
|
"final_status": "success",
|
|
"turns": [
|
|
{
|
|
"index": 0,
|
|
"role": "user",
|
|
"content": "{\"intent\": \"order_status\", \"order_id\": \"ORD-58-B\", \"sku\": \"SKU-42\"}"
|
|
},
|
|
{
|
|
"index": 1,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "query_order",
|
|
"method": "GET",
|
|
"path": "/orders/ORD-58-B",
|
|
"request": null,
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.895,
|
|
"response": {
|
|
"order_id": "ORD-58-B",
|
|
"status": "paid",
|
|
"sku": "SKU-42"
|
|
}
|
|
},
|
|
{
|
|
"index": 2,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "check_stock",
|
|
"method": "GET",
|
|
"path": "/inventory/SKU-42",
|
|
"request": null,
|
|
"http_status": null,
|
|
"status": "error",
|
|
"latency_ms": 102.492,
|
|
"error": "TimeoutError: timed out"
|
|
},
|
|
{
|
|
"index": 3,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "check_stock",
|
|
"method": "GET",
|
|
"path": "/inventory/SKU-42?degraded=1",
|
|
"request": null,
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 1.493,
|
|
"response": {
|
|
"sku": "SKU-42",
|
|
"stock": 12,
|
|
"source": "cache",
|
|
"degraded": true
|
|
}
|
|
},
|
|
{
|
|
"index": 4,
|
|
"role": "tool",
|
|
"module": "http_order_system",
|
|
"tool": "notify_user",
|
|
"method": "POST",
|
|
"path": "/notifications",
|
|
"request": {
|
|
"status": "success"
|
|
},
|
|
"http_status": 200,
|
|
"status": "success",
|
|
"latency_ms": 0.843,
|
|
"response": {
|
|
"sent": true,
|
|
"status": "success"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|