liqiang
b119135836
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
97 lines
4.0 KiB
JSON
97 lines
4.0 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"experiment": "10-3",
|
|
"status": "completed",
|
|
"started_at": "2026-08-03T02:28:29Z",
|
|
"finished_at": "2026-08-03T03:20:52Z",
|
|
"source_repository": "https://github.com/19PINE-AI/TalkAct",
|
|
"source_commit": "7d70007f72d45ddfc1a14e8e229b6d444e4919a2",
|
|
"source_hashes": {
|
|
"requirements.txt": "18a53ec32240d9a141728eee0f223530761ca55c6fc4cc5c60570802a6248dc8",
|
|
"bench/run_bench.py": "397b8f29c2494dab4946f7503a1cd16106fcb6badd37205a317701f223a8764d",
|
|
"bench/analyze.py": "431f18fcf1037073285d184cfa7b1e7d38604be07b5d38218cd090c11f92ee4c",
|
|
"bench/checkers.py": "dd37efc09bd6da0b16c709a4e5f43d067d926d7626fa86f291db685230edf657",
|
|
"bench/tasks.py": "9fd0df7316ae7f5fa343b59f339c087a8b10d35091e66c1b05841f1f845068aa",
|
|
"envs/app.py": "8918f11ad7642a6c5cfba80edcdd3848303aac177959bf759a71a9fb1b61047b",
|
|
"src/cuv/runner.py": "81408c9fc69494bd30a28926bcafb965537f21b82a1fa43436ab94a5577c89a9",
|
|
"src/cuv/slow_agent.py": "8932fd50c91d2581a054bf84652405d2da806bb7363c813977bcf96d716bb5ca",
|
|
"src/cuv/fast_agent.py": "d001df3846cf3f79faff08cb61edc9b902f480f7e7fa3af3621440f6d658bdf7",
|
|
"src/cuv/simuser.py": "d5faf3f7fbaf1e014b1d97456f98a1d034002af2852547ce60701805be908a64",
|
|
"src/cuv/browser.py": "71eaf8eac9cd208057a2028430aafe334f9c1bfe1c23a07254ce7a8db603f9a7",
|
|
"src/cuv/shared.py": "88b2b1d09b4596bd713c21919cd1c878311213d88988e2f404645bb11cd3f88c"
|
|
},
|
|
"command": [
|
|
"python",
|
|
"bench/run_bench.py",
|
|
"--tasks",
|
|
"forms-insurance",
|
|
"booking-flight",
|
|
"webmail-report",
|
|
"meeting-helper",
|
|
"--conditions",
|
|
"duplex",
|
|
"strawman",
|
|
"--seeds",
|
|
"2"
|
|
],
|
|
"campaign": {
|
|
"tasks": [
|
|
"forms-insurance",
|
|
"booking-flight",
|
|
"webmail-report",
|
|
"meeting-helper"
|
|
],
|
|
"conditions": [
|
|
"duplex",
|
|
"strawman"
|
|
],
|
|
"repetitions_labeled_as_seeds": 2,
|
|
"expected_episodes": 16,
|
|
"attempted_episodes": 16,
|
|
"retained_episode_logs": 16,
|
|
"fresh_results_directory": true,
|
|
"skipped_preexisting_episodes": 0
|
|
},
|
|
"providers": {
|
|
"fast_agent": {
|
|
"provider": "Anthropic API",
|
|
"requested_model": "claude-haiku-4-5",
|
|
"preflight_observed_model": "claude-haiku-4-5-20251001",
|
|
"preflight_message_id": "msg_011Cdf29passvJo1sqZSq35c",
|
|
"preflight_usage": {
|
|
"input_tokens": 9,
|
|
"output_tokens": 4
|
|
}
|
|
},
|
|
"slow_agent": {
|
|
"provider": "Anthropic API",
|
|
"requested_model": "claude-opus-4-8",
|
|
"preflight_observed_model": "claude-opus-4-8",
|
|
"preflight_message_id": "msg_011Cdf29tYkYFEzenyAT11t1",
|
|
"preflight_usage": {
|
|
"input_tokens": 11,
|
|
"output_tokens": 4
|
|
}
|
|
}
|
|
},
|
|
"caller_deviation": {
|
|
"reason": "The configured Gemini API key returned 400 API_KEY_INVALID during preflight.",
|
|
"environment_override": "CUV_USER_MODEL",
|
|
"provider": "Anthropic API",
|
|
"model": "claude-sonnet-4-5-20250929",
|
|
"impact": "This source-supported override preserves the benchmark topology but removes the default cross-provider caller and may introduce same-family bias. Results must not be silently pooled with the upstream default-Gemini configuration."
|
|
},
|
|
"postprocessing": {
|
|
"script": "bench/analyze.py",
|
|
"judge_cache_source": "The probe grades produced during the exact run_bench.py campaign were mapped to the corresponding episode filenames before running analyze.py, avoiding a second nondeterministic judge pass.",
|
|
"aggregate": "aggregate.json",
|
|
"per_episode": "per_episode.json"
|
|
},
|
|
"limitations": [
|
|
"The pinned runner uses --seeds to label repeated episodes but does not pass a deterministic random seed into EpisodeRunner or provider calls.",
|
|
"The pinned source retains fast/slow model labels and aggregate token usage per episode, but not individual provider response IDs.",
|
|
"The pinned source does not retain simulated-caller token usage.",
|
|
"meeting-helper success and partial fields are intentionally null; its four answers are evaluated by the retained probe judge."
|
|
]
|
|
}
|