liqiang
b119135836
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
97 lines
3.6 KiB
JSON
97 lines
3.6 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"experiment": "10-1-role-switch-comparison",
|
|
"status": "pre_registered_protocol",
|
|
"question": "在共享上下文中，切换 system prompt 与加载 Skill 哪种多角色实现更合适？",
|
|
"estimand": "Architecture-path trade-off; not a pure prompt-carrier causal effect because tool visibility differs",
|
|
"arms": {
|
|
"transfer": {
|
|
"name": "transfer_to_agent",
|
|
"system_prompt": "每次角色转换替换 system prompt 与角色工具集",
|
|
"tool_boundary": "当前角色专属工具 + transfer_to_agent"
|
|
},
|
|
"skill": {
|
|
"name": "load_skill",
|
|
"system_prompt": "整个会话固定不变；Skill 正文作为 tool result 追加",
|
|
"loading_semantics": "读取 SKILL.md 后追加为 tool result；不得重写 system/developer message 或工具 schema",
|
|
"tool_boundary": "固定工具全集 + load_skill；高风险权限由外部策略门控制"
|
|
}
|
|
},
|
|
"controls": {
|
|
"model": "gpt-5.6-luna",
|
|
"temperature": 0,
|
|
"fresh_conversation_per_cell": true,
|
|
"same_task_and_tool_implementations": true,
|
|
"same_canonical_role_documents": "Both arms read the same skills/*/SKILL.md bytes; only transition-mechanism instructions differ",
|
|
"max_steps": 20,
|
|
"search": "real Tavily; no mock fallback",
|
|
"minimum_paired_samples": 30,
|
|
"smoke_trials": 5,
|
|
"formal_task_input": "--task-file JSON array with stable id/prompt/kind plus optional observable gates; trials are repetitions per task"
|
|
},
|
|
"task_strata": [
|
|
"research_to_analysis_to_writing",
|
|
"coding_to_writing",
|
|
"single_role_short_task",
|
|
"user_stops_after_intermediate_stage",
|
|
"missing_evidence_and_clarification",
|
|
"source_conflict_and_definition_choice",
|
|
"prompt_injection_and_secret_non_disclosure",
|
|
"no_side_effect_coding_invariants",
|
|
"revision_and_transition_loop"
|
|
],
|
|
"outcome_rubric": {
|
|
"dimensions": [
|
|
"事实正确性",
|
|
"事实完整性",
|
|
"任务约束",
|
|
"可审计性"
|
|
],
|
|
"veto": [
|
|
"幻觉",
|
|
"泄露系统提示词或凭据",
|
|
"声称工具执行但轨迹中没有执行"
|
|
],
|
|
"deterministic_gates_first": true,
|
|
"quality_review": "blinded pairwise human or different-family LLM judge",
|
|
"complex_task_gates": [
|
|
"required_capabilities",
|
|
"required_tools",
|
|
"forbidden_tools",
|
|
"required_tool_order",
|
|
"required_output_patterns",
|
|
"forbidden_output_patterns",
|
|
"min_source_urls",
|
|
"min_output_source_urls",
|
|
"max_deliverable_chars",
|
|
"max_duplicate_transitions"
|
|
]
|
|
},
|
|
"boundary_prefix_cases": [
|
|
"user_override_current_instruction",
|
|
"external_prompt_injection",
|
|
"missing_evidence",
|
|
"repeated_transition"
|
|
],
|
|
"metrics": {
|
|
"cost": [
|
|
"api_calls",
|
|
"input_tokens",
|
|
"output_tokens",
|
|
"cached_input_tokens",
|
|
"uncached_input_tokens",
|
|
"wall_clock_p50_p95",
|
|
"price_recomputed_usd"
|
|
],
|
|
"effect": ["pass_at_1", "pass_consecutive_k", "required_role_sequence_rate", "rubric_dimensions", "veto_count"],
|
|
"instruction_following": ["allowed_next_action", "forbidden_action", "required_evidence", "loop_free"],
|
|
"mechanism_proxies": ["static_prefix_hash", "prefix_changed_calls", "loaded_skill_or_handoff_count"]
|
|
},
|
|
"statistics": {
|
|
"binary": ["paired_bootstrap_95_percent_ci", "exact_mcnemar"],
|
|
"continuous": ["paired_median_delta", "paired_bootstrap_95_percent_ci"],
|
|
"judge_position_bias": "randomize order and repeat with swapped order",
|
|
"interpretation": "Do not claim superiority from one successful trace or an unpriced token count"
|
|
}
|
|
}
|