liqiang
b119135836
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
157 lines
15 KiB
JSON
157 lines
15 KiB
JSON
{
|
|
"artifact_type": "agent_control_code_patch",
|
|
"failure_cluster": [
|
|
{
|
|
"cluster_id": "charge_card:PAYMENT_DECLINED",
|
|
"tool": "charge_card",
|
|
"error_code": "PAYMENT_DECLINED",
|
|
"source_case_ids": [
|
|
"run-payment-001",
|
|
"run-payment-002"
|
|
],
|
|
"cross_trajectory_support": 2,
|
|
"total_redundant_calls": 5
|
|
}
|
|
],
|
|
"source_trajectories": [
|
|
{
|
|
"id": "run-payment-001",
|
|
"trajectory_sha256": "4a0b38498b006bf988279b0ff7984f04245db21ee0668986a49f3ca3f4969008",
|
|
"evidence": "The same declined payment was submitted four times."
|
|
},
|
|
{
|
|
"id": "run-payment-002",
|
|
"trajectory_sha256": "6407eb8005fb39702005d553963405329607b34e67acbb18901c79060dbe1e59",
|
|
"evidence": "A second task repeated the identical non-retryable call three times."
|
|
}
|
|
],
|
|
"inferred_root_cause": "The deterministic control policy ignores retryable=false and does not open the circuit for permanent errors. The root cause belongs in retry/circuit-breaker code, not a prompt.",
|
|
"target_component": "retry_and_circuit_breaker_control",
|
|
"target_file": "stable/retry_policy.py",
|
|
"code_diff": "--- stable/retry_policy.py\n+++ candidate/retry_policy.py\n@@ -1,16 +1,18 @@\n-\"\"\"Stable version 1.0.0. It intentionally contains the reproduced bug.\"\"\"\n-\n-VERSION = \"1.0.0\"\n+VERSION = \"1.0.1\"\n MAX_RETRIES = 3\n CIRCUIT_BREAKER_THRESHOLD = 5\n-NON_RETRYABLE_CODES = {\"AUTH_DENIED\", \"INVALID_ARGUMENT\"}\n+NON_RETRYABLE_CODES = {\"AUTH_DENIED\", \"INVALID_ARGUMENT\", \"PAYMENT_DECLINED\"}\n \n \n def should_retry(error_code, retryable, attempt):\n \"\"\"Return whether another tool call should be attempted.\"\"\"\n+ if not retryable or error_code in NON_RETRYABLE_CODES:\n+ return False\n return attempt < MAX_RETRIES\n \n \n def should_open_circuit(consecutive_failures, *, error_code=\"\", retryable=True):\n \"\"\"Open after repeated failures.\"\"\"\n+ if not retryable or error_code in NON_RETRYABLE_CODES:\n+ return consecutive_failures >= 1\n return consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD\n",
|
|
"impact_prediction": {
|
|
"non_retryable_calls": {
|
|
"before": "up to 4",
|
|
"after": 1
|
|
},
|
|
"temporary_timeout_recovery_rate": {
|
|
"before": 1.0,
|
|
"after": 1.0
|
|
}
|
|
},
|
|
"expected_fix": [
|
|
"non-retryable tool-call attempts fall to one",
|
|
"the circuit opens on the first permanent failure"
|
|
],
|
|
"potential_regressions": [
|
|
"temporary timeouts stop retrying",
|
|
"the five-failure circuit threshold changes for retryable errors",
|
|
"public function signatures change"
|
|
],
|
|
"stable_version": "fcd37db6ebb1",
|
|
"stable_sha256": "fcd37db6ebb19ed2e2492f086365a9f8addfcc4410242151185a1481d130041a",
|
|
"candidate_version": "801604c8b4eb",
|
|
"candidate_sha256": "801604c8b4ebe0611dc0d78c4b0131736341bcac990df6ad817607b8486d402b",
|
|
"rollback_version": "fcd37db6ebb1",
|
|
"rollback_sha256": "fcd37db6ebb19ed2e2492f086365a9f8addfcc4410242151185a1481d130041a",
|
|
"diff": "--- stable/retry_policy.py\n+++ candidate/retry_policy.py\n@@ -1,16 +1,18 @@\n-\"\"\"Stable version 1.0.0. It intentionally contains the reproduced bug.\"\"\"\n-\n-VERSION = \"1.0.0\"\n+VERSION = \"1.0.1\"\n MAX_RETRIES = 3\n CIRCUIT_BREAKER_THRESHOLD = 5\n-NON_RETRYABLE_CODES = {\"AUTH_DENIED\", \"INVALID_ARGUMENT\"}\n+NON_RETRYABLE_CODES = {\"AUTH_DENIED\", \"INVALID_ARGUMENT\", \"PAYMENT_DECLINED\"}\n \n \n def should_retry(error_code, retryable, attempt):\n \"\"\"Return whether another tool call should be attempted.\"\"\"\n+ if not retryable or error_code in NON_RETRYABLE_CODES:\n+ return False\n return attempt < MAX_RETRIES\n \n \n def should_open_circuit(consecutive_failures, *, error_code=\"\", retryable=True):\n \"\"\"Open after repeated failures.\"\"\"\n+ if not retryable or error_code in NON_RETRYABLE_CODES:\n+ return consecutive_failures >= 1\n return consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD\n",
|
|
"patch_size": {
|
|
"added_lines": 6,
|
|
"deleted_lines": 4,
|
|
"changed_lines": 10
|
|
},
|
|
"checks": {
|
|
"static_compile": true,
|
|
"public_api_compatible": true,
|
|
"security_scan": true,
|
|
"failure_replay": true,
|
|
"nonretryable_circuit": true,
|
|
"temporary_recovery": true,
|
|
"old_task_regression": true,
|
|
"canary_ready": true,
|
|
"rollback_ready": true,
|
|
"protected_surfaces_unchanged": true
|
|
},
|
|
"failed_checks": [],
|
|
"canary_gate": {
|
|
"eligible": true,
|
|
"scope": "shadow traffic only; stable remains unchanged",
|
|
"rollback_trigger": "any non-retryable repeat or temporary-recovery regression"
|
|
},
|
|
"rollback_gate": {
|
|
"artifact_hash_matches_stable": true,
|
|
"rollback_version": "fcd37db6ebb1"
|
|
},
|
|
"provenance": {
|
|
"generator": "real_llm_coding_agent",
|
|
"model": "doubao-seed-1-6-250615",
|
|
"provider": "ark",
|
|
"seed": 8501,
|
|
"api_calls": 1,
|
|
"receipt": {
|
|
"backend": {
|
|
"provider": "ark",
|
|
"endpoint": "https://ark.cn-beijing.volces.com/api/v3/chat/completions",
|
|
"credential_env": "ARK_API_KEY",
|
|
"model": "doubao-seed-1-6-250615",
|
|
"credential_value_recorded": false
|
|
},
|
|
"request": {
|
|
"model": "doubao-seed-1-6-250615",
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "You are the Coding Agent in a controlled self-modification pipeline.\n\nModify only the supplied retry-policy module. Preserve both public function\nsignatures and retry behavior for temporary failures. Permanent failures\n(retryable=false or a listed permanent code) must not be retried and must open\nthe circuit on the first occurrence. Update VERSION to a candidate version.\nDo not import modules, access files, or alter validation/release logic.\n\nBefore the source, predict the intended impact. Return JSON only:\n{\"impact_prediction\": {\"non_retryable_calls\": {\"before\": \"up to 4\", \"after\": 1},\n\"temporary_timeout_recovery_rate\": {\"before\": 1.0, \"after\": 1.0}},\n\"source\": \"the complete Python module\"}\n\nFailure diagnosis:\n{\n \"change_required\": true,\n \"target\": \"stable/retry_policy.py\",\n \"target_component\": \"retry_and_circuit_breaker_control\",\n \"source_case_ids\": [\n \"run-payment-001\",\n \"run-payment-002\"\n ],\n \"source_trajectories\": [\n {\n \"id\": \"run-payment-001\",\n \"trajectory_sha256\": \"4a0b38498b006bf988279b0ff7984f04245db21ee0668986a49f3ca3f4969008\",\n \"evidence\": \"The same declined payment was submitted four times.\"\n },\n {\n \"id\": \"run-payment-002\",\n \"trajectory_sha256\": \"6407eb8005fb39702005d553963405329607b34e67acbb18901c79060dbe1e59\",\n \"evidence\": \"A second task repeated the identical non-retryable call three times.\"\n }\n ],\n \"patterns\": [\n {\n \"cluster_id\": \"charge_card:PAYMENT_DECLINED\",\n \"tool\": \"charge_card\",\n \"error_code\": \"PAYMENT_DECLINED\",\n \"source_case_ids\": [\n \"run-payment-001\",\n \"run-payment-002\"\n ],\n \"cross_trajectory_support\": 2,\n \"total_redundant_calls\": 5\n }\n ],\n \"reason\": \"The deterministic control policy ignores retryable=false and does not open the circuit for permanent errors. The root cause belongs in retry/circuit-breaker code, not a prompt.\",\n \"change_contract\": {\n \"expected_fix\": [\n \"non-retryable tool-call attempts fall to one\",\n \"the circuit opens on the first permanent failure\"\n ],\n \"potential_regressions\": [\n \"temporary timeouts stop retrying\",\n \"the five-failure circuit threshold changes for retryable errors\",\n \"public function signatures change\"\n ]\n }\n}\n\nPreviously rejected candidates (do not repeat their failure):\n[\n {\n \"candidate_sha256\": \"824bbfb244a7b17ac835706be588542a63ad03e0d737d6c9d5da719c969652d2\",\n \"failed_checks\": [\n \"temporary_recovery\",\n \"old_task_regression\",\n \"canary_ready\"\n ],\n \"rejection_reason\": \"failed gates: temporary_recovery, old_task_regression, canary_ready\",\n \"failure\": \"disabled temporary-timeout retries\"\n }\n]\n\nStable module:\n\"\"\"Stable version 1.0.0. It intentionally contains the reproduced bug.\"\"\"\n\nVERSION = \"1.0.0\"\nMAX_RETRIES = 3\nCIRCUIT_BREAKER_THRESHOLD = 5\nNON_RETRYABLE_CODES = {\"AUTH_DENIED\", \"INVALID_ARGUMENT\"}\n\n\ndef should_retry(error_code, retryable, attempt):\n \"\"\"Return whether another tool call should be attempted.\"\"\"\n return attempt < MAX_RETRIES\n\n\ndef should_open_circuit(consecutive_failures, *, error_code=\"\", retryable=True):\n \"\"\"Open after repeated failures.\"\"\"\n return consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD\n\n"
|
|
}
|
|
],
|
|
"temperature": 0,
|
|
"seed": 8501,
|
|
"max_tokens": 1400,
|
|
"response_format": {
|
|
"type": "json_object"
|
|
}
|
|
},
|
|
"response": {
|
|
"id": "0217853450409166bc656bbd49cfaee971fadc31892ba5b3aa2b5",
|
|
"choices": [
|
|
{
|
|
"finish_reason": "stop",
|
|
"index": 0,
|
|
"message": {
|
|
"content": "{\"impact_prediction\": {\"non_retryable_calls\": {\"before\": \"up to 4\", \"after\": 1}, \"temporary_timeout_recovery_rate\": {\"before\": 1.0, \"after\": 1.0}}, \"source\": \"VERSION = \\\"1.0.1\\\"\\nMAX_RETRIES = 3\\nCIRCUIT_BREAKER_THRESHOLD = 5\\nNON_RETRYABLE_CODES = {\\\"AUTH_DENIED\\\", \\\"INVALID_ARGUMENT\\\", \\\"PAYMENT_DECLINED\\\"}\\n\\n\\ndef should_retry(error_code, retryable, attempt):\\n \\\"\\\"\\\"Return whether another tool call should be attempted.\\\"\\\"\\\"\\n if not retryable or error_code in NON_RETRYABLE_CODES:\\n return False\\n return attempt < MAX_RETRIES\\n\\n\\ndef should_open_circuit(consecutive_failures, *, error_code=\\\"\\\", retryable=True):\\n \\\"\\\"\\\"Open after repeated failures.\\\"\\\"\\\"\\n if not retryable or error_code in NON_RETRYABLE_CODES:\\n return consecutive_failures >= 1\\n return consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD\"}",
|
|
"role": "assistant",
|
|
"reasoning_content": "Okay, let's tackle this problem. The user wants me to modify the retry-policy module to fix the issue where non-retryable errors are being retried multiple times, and the circuit isn't opening on the first permanent failure. \n\nFirst, I need to understand the current behavior. From the failure diagnosis, the problem is that even when an error is non-retryable (retryable=false) or has a permanent error code, the retry policy is still allowing retries. For example, in the cases provided, PAYMENT_DECLINED errors were retried multiple times, leading to redundant calls.\n\nLooking at the existing code:\n\nThe should_retry function currently checks if the attempt is less than MAX_RETRIES (which is 3). So even if the error is non-retryable, it would retry up to 3 times, leading to 4 total attempts (initial + 3 retries). That's why in the examples, there were 4 and 3 calls respectively.\n\nThe should_open_circuit function opens the circuit only after consecutive_failures reach CIRCUIT_BREAKER_THRESHOLD (5), regardless of the error type. So permanent failures aren't triggering the circuit to open immediately.\n\nThe change needs to ensure that non-retryable errors (either retryable=false or in NON_RETRYABLE_CODES) are not retried. So should_retry should return False in those cases. Also, when such an error occurs, the circuit should open on the first occurrence.\n\nSo modifying should_retry:\n\nOriginal: return attempt < MAX_RETRIES\n\nNew logic: Only retry if retryable is True, the error_code is not in NON_RETRYABLE_CODES, and attempt < MAX_RETRIES.\n\nWait, the parameters for should_retry are error_code, retryable, attempt. So the function should check if retryable is False, then return False. Also, if error_code is in NON_RETRYABLE_CODES, return False. Otherwise, check if attempt < MAX_RETRIES.\n\nSo the should_retry function should be:\n\ndef should_retry(error_code, retryable, attempt):\n if not retryable or error_code in NON_RETRYABLE_CODES:\n return False\n return attempt < MAX_RETRIES\n\nThat way, if either retryable is False or the error code is non-retryable, it won't retry. So the initial attempt is 1, and no retries, so total 1 call. Which fixes the first part of the problem.\n\nFor the circuit breaker: should_open_circuit needs to open on the first permanent failure. So if the error is non-retryable (retryable=False or error_code in NON_RETRYABLE_CODES), then consecutive_failures >=1 should trigger the circuit. But how does the current function get the error info? The parameters are consecutive_failures, and keyword args error_code and retryable.\n\nSo modifying should_open_circuit:\n\nOriginal: return consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD\n\nNew logic: If the error is a permanent failure (retryable=False or error_code in NON_RETRYABLE_CODES), then open the circuit if consecutive_failures >=1. Otherwise, use the existing threshold of 5.\n\nSo:\n\ndef should_open_circuit(consecutive_failures, *, error_code=\"\", retryable=True):\n if not retryable or error_code in NON_RETRYABLE_CODES:\n return consecutive_failures >= 1\n return consecutive_failures >= CIRCUIT_BREAKER_THRESHOLD\n\nThis way, for permanent errors, even one failure opens the circuit. For temporary ones, it still takes 5.\n\nAlso, the VERSION needs to be updated. The original is 1.0.0, so maybe 1.0.1 or 1.1.0. Since it's a bug fix, 1.0.1 makes sense.\n\nPutting it all together:\n\nUpdate VERSION to \"1.0.1\".\n\nModify should_retry to check retryable and error_code.\n\nModify should_open_circuit to check for permanent errors and open on first failure.\n\nTesting for possible regressions: The problem mentions that temporary timeouts should still retry. So if a temporary error (retryable=True and not in NON_RETRYABLE_CODES) occurs, the retries should still happen. The original MAX_RETRIES is 3, so 4 attempts total. The new code preserves that, so temporary_timeout_recovery_rate remains 1.0.\n\nSo the impact prediction would be non_retryable_calls before up to 4 (initial + 3 retries), after 1. Temporary timeout recovery rate remains 1.0.\n\nThus, the JSON impact prediction is as specified.\n\nNow, the source code with these changes.\n"
|
|
}
|
|
}
|
|
],
|
|
"created": 1785345078,
|
|
"model": "doubao-seed-1-6-250615",
|
|
"object": "chat.completion",
|
|
"service_tier": "default",
|
|
"usage": {
|
|
"completion_tokens": 1276,
|
|
"prompt_tokens": 1128,
|
|
"total_tokens": 2404,
|
|
"completion_tokens_details": {
|
|
"reasoning_tokens": 1019
|
|
},
|
|
"prompt_tokens_details": {
|
|
"cached_tokens": 0
|
|
}
|
|
}
|
|
},
|
|
"request_sha256": "fe5695eec21fe9f9674035f307d8e5f6bfacafa244a8184eb734188d6be405af",
|
|
"response_sha256": "ba47b7677546e460af60eac62f20d263c5fbb4432fdffc40cc00ad01d6f6f034",
|
|
"elapsed_seconds": 37.571053,
|
|
"usage": {
|
|
"prompt_tokens": 1128,
|
|
"completion_tokens": 1276,
|
|
"total_tokens": 2404,
|
|
"provider_reported_cost_usd": null,
|
|
"cost_qualification": "provider did not expose monetary cost; no price was guessed"
|
|
}
|
|
}
|
|
},
|
|
"decision": "release_to_canary",
|
|
"rejection_reason": null
|
|
} |