liqiang/ai-agent-book
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
Details
dependency resolution / resolve (3.11) (push) Canceled after 0s
Details
dependency resolution / resolve (3.13) (push) Canceled after 0s
Details
deploy-pages / build (push) Canceled after 0s
Details
deploy-pages / deploy (push) Canceled after 0s
Details
i18n consistency check / check (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
Details
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
Details
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
Details
web-search-agent tests / test (push) Canceled after 0s
Details
web-search-agent tests / agentbook (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
liqiang
2026-08-20 13:12:50 +00:00
commit b119135836
10275 changed files with 3284984 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
chapter7/elo-leaderboard/validation/runs/exp7-7-arena-20260731-v1/bradley_terry.json
+905
View File
@@ -0,0 +1,905 @@
[
{
"lower_ci": 1199.4356201692,
"model": "chatgpt-4o-latest",
"rank": 1,
"rating": 1202.8722324768,
"upper_ci": 1207.3805726125
},
{
"lower_ci": 1182.767114327,
"model": "gemini-1.5-pro-exp-0801",
"rank": 2,
"rating": 1187.3727762743,
"upper_ci": 1191.7564879767
},
{
"lower_ci": 1173.1270556478,
"model": "gpt-4o-2024-05-13",
"rank": 3,
"rating": 1174.5475812185,
"upper_ci": 1177.0403252386
},
{
"lower_ci": 1161.1978031325,
"model": "gpt-4o-mini-2024-07-18",
"rank": 4,
"rating": 1162.9712796813,
"upper_ci": 1166.568334134
},
{
"lower_ci": 1156.0224735749,
"model": "claude-3-5-sonnet-20240620",
"rank": 5,
"rating": 1159.6692177384,
"upper_ci": 1161.8458744744
},
{
"lower_ci": 1153.6447817793,
"model": "gemini-advanced-0514",
"rank": 6,
"rating": 1155.6305094344,
"upper_ci": 1157.1030226933
},
{
"lower_ci": 1150.4614200682,
"model": "llama-3.1-405b-instruct",
"rank": 7,
"rating": 1153.3950109278,
"upper_ci": 1157.0848375454
},
{
"lower_ci": 1146.348878889,
"model": "gpt-4o-2024-08-06",
"rank": 8,
"rating": 1150.7428051987,
"upper_ci": 1156.3245456986
},
{
"lower_ci": 1146.7772301944,
"model": "gemini-1.5-pro-api-0514",
"rank": 9,
"rating": 1149.1265569601,
"upper_ci": 1151.5010539806
},
{
"lower_ci": 1143.6682527667,
"model": "gemini-1.5-pro-api-0409-preview",
"rank": 10,
"rating": 1146.1756037638,
"upper_ci": 1147.8895040945
},
{
"lower_ci": 1142.3990049162,
"model": "gpt-4-turbo-2024-04-09",
"rank": 11,
"rating": 1144.9187340632,
"upper_ci": 1146.8362809167
},
{
"lower_ci": 1138.1303324947,
"model": "gpt-4-1106-preview",
"rank": 12,
"rating": 1139.5004505155,
"upper_ci": 1140.97644635
},
{
"lower_ci": 1136.3852091186,
"model": "mistral-large-2407",
"rank": 13,
"rating": 1139.3566231011,
"upper_ci": 1143.8960727747
},
{
"lower_ci": 1135.0624903381,
"model": "athene-70b-0725",
"rank": 14,
"rating": 1138.2016223593,
"upper_ci": 1141.397916447
},
{
"lower_ci": 1135.0278139027,
"model": "claude-3-opus-20240229",
"rank": 15,
"rating": 1136.5739464324,
"upper_ci": 1137.5663039742
},
{
"lower_ci": 1131.7219814044,
"model": "llama-3.1-70b-instruct",
"rank": 16,
"rating": 1134.8942150397,
"upper_ci": 1139.3028524197
},
{
"lower_ci": 1132.1800230236,
"model": "gpt-4-0125-preview",
"rank": 17,
"rating": 1133.974390087,
"upper_ci": 1135.7552442263
},
{
"lower_ci": 1127.3300289872,
"model": "yi-large-preview",
"rank": 18,
"rating": 1128.1692229165,
"upper_ci": 1131.371743138
},
{
"lower_ci": 1111.850158929,
"model": "reka-core-20240722",
"rank": 19,
"rating": 1117.1464671073,
"upper_ci": 1121.9537941313
},
{
"lower_ci": 1114.0244965884,
"model": "gemini-1.5-flash-api-0514",
"rank": 20,
"rating": 1116.6161359294,
"upper_ci": 1118.5223962538
},
{
"lower_ci": 1103.6704090106,
"model": "deepseek-v2-api-0628",
"rank": 21,
"rating": 1107.2808888587,
"upper_ci": 1112.0417779712
},
{
"lower_ci": 1103.9432951297,
"model": "gemma-2-27b-it",
"rank": 22,
"rating": 1105.9132519337,
"upper_ci": 1108.0293102426
},
{
"lower_ci": 1099.3459906048,
"model": "deepseek-coder-v2-0724",
"rank": 23,
"rating": 1105.3954927337,
"upper_ci": 1113.0353361228
},
{
"lower_ci": 1099.9821845459,
"model": "yi-large",
"rank": 24,
"rating": 1102.4018397329,
"upper_ci": 1104.8525831946
},
{
"lower_ci": 1094.5198632757,
"model": "nemotron-4-340b-instruct",
"rank": 25,
"rating": 1098.0765768899,
"upper_ci": 1100.3082488796
},
{
"lower_ci": 1092.3225621341,
"model": "bard-jan-24-gemini-pro",
"rank": 26,
"rating": 1096.589845604,
"upper_ci": 1104.5333605578
},
{
"lower_ci": 1089.9500461084,
"model": "glm-4-0520",
"rank": 27,
"rating": 1095.5341569364,
"upper_ci": 1100.1005750187
},
{
"lower_ci": 1093.6731280724,
"model": "llama-3-70b-instruct",
"rank": 28,
"rating": 1094.9281802329,
"upper_ci": 1095.7283536945
},
{
"lower_ci": 1088.0897564085,
"model": "claude-3-sonnet-20240229",
"rank": 29,
"rating": 1090.1434308802,
"upper_ci": 1091.5131310821
},
{
"lower_ci": 1086.224736458,
"model": "reka-core-20240501",
"rank": 30,
"rating": 1088.4321613308,
"upper_ci": 1090.272779249
},
{
"lower_ci": 1082.4844014079,
"model": "reka-flash-20240722",
"rank": 31,
"rating": 1088.2114217704,
"upper_ci": 1094.1409244278
},
{
"lower_ci": 1076.2505646464,
"model": "command-r-plus",
"rank": 32,
"rating": 1078.6392855764,
"upper_ci": 1080.5241685262
},
{
"lower_ci": 1072.978332193,
"model": "gemma-2-9b-it",
"rank": 33,
"rating": 1076.1200490762,
"upper_ci": 1078.284506667
},
{
"lower_ci": 1073.5327980749,
"model": "qwen2-72b-instruct",
"rank": 34,
"rating": 1075.9397511888,
"upper_ci": 1077.9102798056
},
{
"lower_ci": 1073.0022184434,
"model": "gpt-4-0314",
"rank": 35,
"rating": 1074.4688060969,
"upper_ci": 1077.3841437021
},
{
"lower_ci": 1069.6849684012,
"model": "qwen-max-0428",
"rank": 36,
"rating": 1072.288830872,
"upper_ci": 1075.2788486953
},
{
"lower_ci": 1067.4064473159,
"model": "glm-4-0116",
"rank": 37,
"rating": 1072.1854778301,
"upper_ci": 1078.0278828583
},
{
"lower_ci": 1065.2785446605,
"model": "claude-3-haiku-20240307",
"rank": 38,
"rating": 1067.2876085746,
"upper_ci": 1068.5180263057
},
{
"lower_ci": 1063.4499078695,
"model": "deepseek-coder-v2",
"rank": 39,
"rating": 1066.5634197273,
"upper_ci": 1071.0372811761
},
{
"lower_ci": 1052.4060997176,
"model": "llama-3.1-8b-instruct",
"rank": 40,
"rating": 1057.2617029177,
"upper_ci": 1062.4851344029
},
{
"lower_ci": 1050.6516231153,
"model": "reka-flash-preview-20240611",
"rank": 41,
"rating": 1053.4286082128,
"upper_ci": 1056.9047978837
},
{
"lower_ci": 1049.878634752,
"model": "gpt-4-0613",
"rank": 42,
"rating": 1051.2288727429,
"upper_ci": 1053.3859970522
},
{
"lower_ci": 1047.658821981,
"model": "qwen1.5-110b-chat",
"rank": 43,
"rating": 1050.1472801797,
"upper_ci": 1054.1870464263
},
{
"lower_ci": 1043.4416581432,
"model": "yi-1.5-34b-chat",
"rank": 44,
"rating": 1046.5798307425,
"upper_ci": 1048.4870877901
},
{
"lower_ci": 1043.9717227179,
"model": "mistral-large-2402",
"rank": 45,
"rating": 1045.8549746499,
"upper_ci": 1048.6558688014
},
{
"lower_ci": 1039.8435107486,
"model": "reka-flash-21b-20240226-online",
"rank": 46,
"rating": 1044.4541273285,
"upper_ci": 1047.5722059799
},
{
"lower_ci": 1038.3457593513,
"model": "llama-3-8b-instruct",
"rank": 47,
"rating": 1040.8943517232,
"upper_ci": 1042.1727089946
},
{
"lower_ci": 1034.3532315209,
"model": "command-r",
"rank": 48,
"rating": 1037.5954245649,
"upper_ci": 1039.801976874
},
{
"lower_ci": 1033.8210405932,
"model": "claude-1",
"rank": 49,
"rating": 1037.2264989242,
"upper_ci": 1039.8445566211
},
{
"lower_ci": 1032.4181357042,
"model": "reka-flash-21b-20240226",
"rank": 50,
"rating": 1036.2083492306,
"upper_ci": 1038.9143450059
},
{
"lower_ci": 1033.5592570692,
"model": "mistral-medium",
"rank": 51,
"rating": 1035.9830773208,
"upper_ci": 1038.0506828269
},
{
"lower_ci": 1033.4716682328,
"model": "mixtral-8x22b-instruct-v0.1",
"rank": 52,
"rating": 1035.6187089038,
"upper_ci": 1037.3385699205
},
{
"lower_ci": 1032.3748800585,
"model": "qwen1.5-72b-chat",
"rank": 53,
"rating": 1035.5734020095,
"upper_ci": 1037.8584457516
},
{
"lower_ci": 1016.6859168483,
"model": "claude-2.0",
"rank": 54,
"rating": 1020.4032512576,
"upper_ci": 1023.0900087939
},
{
"lower_ci": 1016.4818650296,
"model": "gemini-pro-dev-api",
"rank": 55,
"rating": 1018.8216738996,
"upper_ci": 1022.0575603351
},
{
"lower_ci": 1013.9644553596,
"model": "gemma-2-2b-it",
"rank": 56,
"rating": 1018.7496539694,
"upper_ci": 1022.6257555787
},
{
"lower_ci": 1009.8189712109,
"model": "zephyr-orpo-141b-A35b-v0.1",
"rank": 57,
"rating": 1017.4768858149,
"upper_ci": 1023.4231756618
},
{
"lower_ci": 1011.7444717478,
"model": "qwen1.5-32b-chat",
"rank": 58,
"rating": 1014.2192047782,
"upper_ci": 1017.4848181067
},
{
"lower_ci": 1009.527805507,
"model": "mistral-next",
"rank": 59,
"rating": 1013.8190347615,
"upper_ci": 1016.756252395
},
{
"lower_ci": 1008.0043265533,
"model": "phi-3-medium-4k-instruct",
"rank": 60,
"rating": 1011.6064259691,
"upper_ci": 1014.8977711664
},
{
"lower_ci": 1003.3664596337,
"model": "claude-2.1",
"rank": 61,
"rating": 1006.9894951694,
"upper_ci": 1008.7578243389
},
{
"lower_ci": 1003.5743630599,
"model": "starling-lm-7b-beta",
"rank": 62,
"rating": 1006.9783218473,
"upper_ci": 1009.6647795891
},
{
"lower_ci": 1003.3560004715,
"model": "gpt-3.5-turbo-0613",
"rank": 63,
"rating": 1004.9730864576,
"upper_ci": 1008.7646121872
},
{
"lower_ci": 1000.3011942689,
"model": "mixtral-8x7b-instruct-v0.1",
"rank": 64,
"rating": 1002.12545233,
"upper_ci": 1004.4555563028
},
{
"lower_ci": 996.8355186436,
"model": "yi-34b-chat",
"rank": 65,
"rating": 1000.1263930743,
"upper_ci": 1002.9767969377
},
{
"lower_ci": 996.5513388826,
"model": "claude-instant-1",
"rank": 66,
"rating": 999.4536204497,
"upper_ci": 1003.0420312018
},
{
"lower_ci": 991.5815494729,
"model": "gemini-pro",
"rank": 67,
"rating": 998.3909699055,
"upper_ci": 1005.6665144072
},
{
"lower_ci": 994.5272838271,
"model": "qwen1.5-14b-chat",
"rank": 68,
"rating": 997.6714948516,
"upper_ci": 1000.3276398407
},
{
"lower_ci": 988.7882364736,
"model": "gpt-3.5-turbo-0314",
"rank": 69,
"rating": 997.4790766208,
"upper_ci": 1005.2920277339
},
{
"lower_ci": 989.2933547328,
"model": "wizardlm-70b",
"rank": 70,
"rating": 995.5069328559,
"upper_ci": 1000.0203225061
},
{
"lower_ci": 992.3677866278,
"model": "gpt-3.5-turbo-0125",
"rank": 71,
"rating": 994.221194786,
"upper_ci": 996.0922832004
},
{
"lower_ci": 989.517752069,
"model": "dbrx-instruct-preview",
"rank": 72,
"rating": 991.6558051925,
"upper_ci": 994.3414735308
},
{
"lower_ci": 988.3935167102,
"model": "phi-3-small-8k-instruct",
"rank": 73,
"rating": 990.3926872181,
"upper_ci": 993.4532484928
},
{
"lower_ci": 980.2003929739,
"model": "tulu-2-dpo-70b",
"rank": 74,
"rating": 986.8619167507,
"upper_ci": 995.6082960114
},
{
"lower_ci": 979.2393563337,
"model": "llama-2-70b-chat",
"rank": 75,
"rating": 982.0382072414,
"upper_ci": 983.1882888775
},
{
"lower_ci": 976.2082709482,
"model": "openchat-3.5-0106",
"rank": 76,
"rating": 980.6486425506,
"upper_ci": 984.7448454329
},
{
"lower_ci": 976.21123294,
"model": "vicuna-33b",
"rank": 77,
"rating": 978.8419077515,
"upper_ci": 983.0972163635
},
{
"lower_ci": 976.387496134,
"model": "snowflake-arctic-instruct",
"rank": 78,
"rating": 978.4458039182,
"upper_ci": 981.5060581663
},
{
"lower_ci": 969.5806601345,
"model": "starling-lm-7b-alpha",
"rank": 79,
"rating": 976.025808172,
"upper_ci": 980.086762811
},
{
"lower_ci": 965.8992419629,
"model": "nous-hermes-2-mixtral-8x7b-dpo",
"rank": 80,
"rating": 975.4699172429,
"upper_ci": 981.307231701
},
{
"lower_ci": 969.0894947313,
"model": "gemma-1.1-7b-it",
"rank": 81,
"rating": 971.847848073,
"upper_ci": 975.4703266199
},
{
"lower_ci": 966.6172052307,
"model": "llama2-70b-steerlm-chat",
"rank": 82,
"rating": 971.6537111318,
"upper_ci": 979.8268878882
},
{
"lower_ci": 961.617441776,
"model": "pplx-70b-online",
"rank": 83,
"rating": 967.4367908935,
"upper_ci": 972.8126406202
},
{
"lower_ci": 959.5692952959,
"model": "openchat-3.5",
"rank": 84,
"rating": 966.0510063396,
"upper_ci": 971.8466170437
},
{
"lower_ci": 956.7287406383,
"model": "deepseek-llm-67b-chat",
"rank": 85,
"rating": 965.1147523375,
"upper_ci": 971.4330625559
},
{
"lower_ci": 956.3934469962,
"model": "openhermes-2.5-mistral-7b",
"rank": 86,
"rating": 963.454411119,
"upper_ci": 968.2145476373
},
{
"lower_ci": 957.6795493747,
"model": "mistral-7b-instruct-v0.2",
"rank": 87,
"rating": 961.195768793,
"upper_ci": 964.3273838331
},
{
"lower_ci": 955.9914563912,
"model": "qwen1.5-7b-chat",
"rank": 88,
"rating": 959.7566395303,
"upper_ci": 963.8434392791
},
{
"lower_ci": 953.2396686903,
"model": "phi-3-mini-4k-instruct-june-2024",
"rank": 89,
"rating": 958.8907631692,
"upper_ci": 964.0199135738
},
{
"lower_ci": 953.2564762147,
"model": "gpt-3.5-turbo-1106",
"rank": 90,
"rating": 955.8557887224,
"upper_ci": 959.2854635955
},
{
"lower_ci": 951.2318268117,
"model": "phi-3-mini-4k-instruct",
"rank": 91,
"rating": 955.5964299136,
"upper_ci": 958.4767635868
},
{
"lower_ci": 939.9598426545,
"model": "dolphin-2.2.1-mistral-7b",
"rank": 92,
"rating": 952.2051167375,
"upper_ci": 962.2676381777
},
{
"lower_ci": 945.2431328588,
"model": "solar-10.7b-instruct-v1.0",
"rank": 93,
"rating": 951.1224233106,
"upper_ci": 958.6524028214
},
{
"lower_ci": 948.6812080723,
"model": "llama-2-13b-chat",
"rank": 94,
"rating": 951.0465511917,
"upper_ci": 953.7276982632
},
{
"lower_ci": 940.2487365654,
"model": "wizardlm-13b",
"rank": 95,
"rating": 945.6170139554,
"upper_ci": 952.3153173674
},
{
"lower_ci": 937.3497020159,
"model": "zephyr-7b-beta",
"rank": 96,
"rating": 941.7856240916,
"upper_ci": 945.7193614882
},
{
"lower_ci": 925.3514960917,
"model": "mpt-30b-chat",
"rank": 97,
"rating": 933.4637447927,
"upper_ci": 942.1936901157
},
{
"lower_ci": 928.7552755121,
"model": "pplx-7b-online",
"rank": 98,
"rating": 932.9740226673,
"upper_ci": 940.2890736975
},
{
"lower_ci": 924.8240945839,
"model": "codellama-34b-instruct",
"rank": 99,
"rating": 931.25075507,
"upper_ci": 936.6721012017
},
{
"lower_ci": 919.9794338309,
"model": "zephyr-7b-alpha",
"rank": 100,
"rating": 930.8024303718,
"upper_ci": 940.3904190626
},
{
"lower_ci": 927.6228696967,
"model": "vicuna-13b",
"rank": 101,
"rating": 930.129806663,
"upper_ci": 933.8873904378
},
{
"lower_ci": 916.3079081804,
"model": "codellama-70b-instruct",
"rank": 102,
"rating": 929.7441814853,
"upper_ci": 939.4638589466
},
{
"lower_ci": 919.0275736817,
"model": "gemma-7b-it",
"rank": 103,
"rating": 925.6063141152,
"upper_ci": 931.2321796245
},
{
"lower_ci": 921.4335696267,
"model": "llama-2-7b-chat",
"rank": 104,
"rating": 925.1226871516,
"upper_ci": 927.4211774328
},
{
"lower_ci": 919.914948524,
"model": "phi-3-mini-128k-instruct",
"rank": 105,
"rating": 925.0941128748,
"upper_ci": 928.953409783
},
{
"lower_ci": 917.0649637172,
"model": "qwen-14b-chat",
"rank": 106,
"rating": 923.0565707958,
"upper_ci": 927.5147584562
},
{
"lower_ci": 910.6316722798,
"model": "falcon-180b-chat",
"rank": 107,
"rating": 922.8642075646,
"upper_ci": 938.170426621
},
{
"lower_ci": 912.8204058608,
"model": "guanaco-33b",
"rank": 108,
"rating": 921.0201740806,
"upper_ci": 929.2566479601
},
{
"lower_ci": 903.2450471973,
"model": "gemma-1.1-2b-it",
"rank": 109,
"rating": 908.7494526802,
"upper_ci": 913.4741035764
},
{
"lower_ci": 899.9846491667,
"model": "stripedhyena-nous-7b",
"rank": 110,
"rating": 905.1534113535,
"upper_ci": 912.9382997405
},
{
"lower_ci": 899.9839053334,
"model": "olmo-7b-instruct",
"rank": 111,
"rating": 903.629872537,
"upper_ci": 908.2570306538
},
{
"lower_ci": 890.8398870858,
"model": "mistral-7b-instruct",
"rank": 112,
"rating": 897.2720855909,
"upper_ci": 902.9292885859
},
{
"lower_ci": 887.0047037055,
"model": "vicuna-7b",
"rank": 113,
"rating": 893.825818387,
"upper_ci": 899.7593423485
},
{
"lower_ci": 887.0036385995,
"model": "palm-2",
"rank": 114,
"rating": 891.3072880052,
"upper_ci": 897.1963496269
},
{
"lower_ci": 872.1578125039,
"model": "gemma-2b-it",
"rank": 115,
"rating": 880.4830688196,
"upper_ci": 884.0952002573
},
{
"lower_ci": 868.4676861275,
"model": "qwen1.5-4b-chat",
"rank": 116,
"rating": 877.0664166393,
"upper_ci": 883.3511645246
},
{
"lower_ci": 846.7609834499,
"model": "koala-13b",
"rank": 117,
"rating": 852.5741703315,
"upper_ci": 859.5370102373
},
{
"lower_ci": 833.5366986653,
"model": "chatglm3-6b",
"rank": 118,
"rating": 843.5467457427,
"upper_ci": 853.6979083485
},
{
"lower_ci": 808.672593832,
"model": "gpt4all-13b-snoozy",
"rank": 119,
"rating": 821.0385947815,
"upper_ci": 835.6716204673
},
{
"lower_ci": 804.1197078456,
"model": "chatglm2-6b",
"rank": 120,
"rating": 815.3737962537,
"upper_ci": 821.0889058237
},
{
"lower_ci": 805.0790145038,
"model": "mpt-7b-chat",
"rank": 121,
"rating": 813.8798502443,
"upper_ci": 823.374846381
},
{
"lower_ci": 803.3055604626,
"model": "RWKV-4-Raven-14B",
"rank": 122,
"rating": 810.0683748343,
"upper_ci": 819.7203248755
},
{
"lower_ci": 781.8369679949,
"model": "alpaca-13b",
"rank": 123,
"rating": 789.0910768147,
"upper_ci": 797.3012759671
},
{
"lower_ci": 778.7113024193,
"model": "oasst-pythia-12b",
"rank": 124,
"rating": 782.6327874047,
"upper_ci": 789.0013974908
},
{
"lower_ci": 760.5962982346,
"model": "chatglm-6b",
"rank": 125,
"rating": 767.7699631784,
"upper_ci": 776.2247501319
},
{
"lower_ci": 749.711207841,
"model": "fastchat-t5-3b",
"rank": 126,
"rating": 756.8214872887,
"upper_ci": 762.6617679978
},
{
"lower_ci": 723.1589751426,
"model": "stablelm-tuned-alpha-7b",
"rank": 127,
"rating": 727.6986691281,
"upper_ci": 737.7996673488
},
{
"lower_ci": 701.8523006848,
"model": "dolly-v2-12b",
"rank": 128,
"rating": 710.1023148692,
"upper_ci": 717.2721178625
},
{
"lower_ci": 679.2645194103,
"model": "llama-13b",
"rank": 129,
"rating": 689.1631346499,
"upper_ci": 694.5590720343
}
]
chapter7/elo-leaderboard/validation/runs/exp7-7-arena-20260731-v1/leaderboard.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 116 KiB

chapter7/elo-leaderboard/validation/runs/exp7-7-arena-20260731-v1/leaderboard_animation.html
+2224
View File
File diff suppressed because it is too large Load Diff
chapter7/elo-leaderboard/validation/runs/exp7-7-arena-20260731-v1/manifest.json
+74
View File
@@ -0,0 +1,74 @@
{
"artifacts": {
"bradley_terry.json": {
"bytes": 19827,
"sha256": "4a861b80c69132fdb3e3d1cfdc9d962444d664623b956595e67798d141ee1c6b"
},
"leaderboard.png": {
"bytes": 118255,
"sha256": "386889f573198987400fef7c7803e0779b95e2fa35cb274d04071fed842c6499"
},
"leaderboard_animation.html": {
"bytes": 54322,
"sha256": "afb9b0426605dcc5accf9e1ea5067dd885ad9a9226884cc341b3430404ca433a"
},
"online_elo.json": {
"bytes": 16885,
"sha256": "41ec9a46bb37457a385d0c4b4036ff8ae3d1affcbae8b9b00689fef494f9f9a6"
},
"rating_history.json": {
"bytes": 354362,
"sha256": "3a5e3193cbbe50bf1de3a9e5d9702f2b594f449300d195674ed0da2a76f72574"
},
"rating_history.png": {
"bytes": 193995,
"sha256": "cb676a82b38a8876bc1346e58f72b6a9d72c8fa5e405a9d22610f6948382fba4"
},
"summary.json": {
"bytes": 10441,
"sha256": "6258914ebdb6be7686985e637903c929ed14535d8572a4b9d42b1256d5cdf261"
},
"win_rate_matrix.json": {
"bytes": 37040,
"sha256": "a6ac9580a8d0290d53adf16458ac777d72e214d0771cd0e72d1b2d8a4d86abd2"
},
"win_rate_matrix.png": {
"bytes": 230883,
"sha256": "db375436646096bfafdad458b53aba54ac156432cf9e8bb7458a2f733c9ae690"
}
},
"experiment": "7-7",
"gates": {
"bradley_terry_official_method_completed": true,
"chronological_online_elo_k4_completed": true,
"d3_animation_saved": true,
"millions_of_blind_votes_loaded": true,
"monthly_history_saved": true,
"official_public_arena_snapshot_hashed": true,
"online_vs_official_method_rank_agreement_observed": true,
"pairwise_empirical_and_predicted_matrix_saved": true,
"static_visualizations_saved": true
},
"input": {
"bytes": 2131976365,
"filename": "arena_data.json",
"sha256": "747c1c937dfa941d5a455a1fd70e2879d29642058da39b97996b977233b9bd1b",
"url": "https://storage.googleapis.com/arena_external_data/public/clean_battle_20240814_public.json"
},
"official_complete": true,
"runtime": {
"numpy": "1.26.4",
"pandas": "2.3.3",
"platform": "macOS-26.3-arm64-arm-64bit",
"python": "3.12.11"
},
"schema_version": 1,
"sources": {
"animation.py": "6839a3173a6a9ee8855901ca3c781fd91de13219b52ce6b285778ab89d926376",
"bradley_terry.py": "030c716284017b14cff204ecd7036020c05554ae0eacf71af5da5c0963848b54",
"optimized_elo.py": "f56b5b4aaba9cda11006bf01742be99fc07ee44ceeed5bd2dee5322a8ce08a57",
"validation/run_experiment.py": "6c668081c4fbf04f9dbc84411a2a40391d7b00c76a90336542cc6a88623e991c",
"validation/validate_evidence.py": "36fe448c94608cfc6a299f21224870357d109d999ba3391ed7d7c71fc1817e62"
},
"status": "passed"
}
chapter7/elo-leaderboard/validation/runs/exp7-7-arena-20260731-v1/online_elo.json
+905
View File
@@ -0,0 +1,905 @@
[
{
"matches": 55654,
"model": "gemini-1.5-pro-api-0409-preview",
"rank": 1,
"rating": 1159.8203895968,
"wins": 33738.5
},
{
"matches": 20071,
"model": "gemini-1.5-pro-exp-0801",
"rank": 2,
"rating": 1127.8877886511,
"wins": 12139.5
},
{
"matches": 14514,
"model": "chatgpt-4o-latest",
"rank": 3,
"rating": 1126.6935165964,
"wins": 8999.0
},
{
"matches": 5655,
"model": "gpt-3.5-turbo-0314",
"rank": 4,
"rating": 1103.9751202512,
"wins": 3789.0
},
{
"matches": 11827,
"model": "bard-jan-24-gemini-pro",
"rank": 5,
"rating": 1094.2118678517,
"wins": 7075.5
},
{
"matches": 21172,
"model": "claude-1",
"rank": 6,
"rating": 1092.7309868534,
"wins": 12081.5
},
{
"matches": 52155,
"model": "gemini-advanced-0514",
"rank": 7,
"rating": 1087.5408038371,
"wins": 30523.0
},
{
"matches": 13604,
"model": "llama-3.1-70b-instruct",
"rank": 8,
"rating": 1087.3529845908,
"wins": 7058.5
},
{
"matches": 77509,
"model": "gpt-4o-2024-05-13",
"rank": 9,
"rating": 1086.7242770205,
"wins": 46951.0
},
{
"matches": 9761,
"model": "gpt-4o-2024-08-06",
"rank": 10,
"rating": 1085.601246973,
"wins": 5323.0
},
{
"matches": 19370,
"model": "gpt-4o-mini-2024-07-18",
"rank": 11,
"rating": 1079.187026245,
"wins": 11082.0
},
{
"matches": 47703,
"model": "claude-3-5-sonnet-20240620",
"rank": 12,
"rating": 1079.0096726066,
"wins": 27107.5
},
{
"matches": 155944,
"model": "claude-3-opus-20240229",
"rank": 13,
"rating": 1074.9078256001,
"wins": 91432.5
},
{
"matches": 11476,
"model": "athene-70b-0725",
"rank": 14,
"rating": 1073.7607622933,
"wins": 6098.0
},
{
"matches": 18790,
"model": "gemini-pro-dev-api",
"rank": 15,
"rating": 1072.3072263884,
"wins": 8728.5
},
{
"matches": 12774,
"model": "claude-2.0",
"rank": 16,
"rating": 1069.530762176,
"wins": 6814.5
},
{
"matches": 10239,
"model": "glm-4-0520",
"rank": 17,
"rating": 1065.2967580553,
"wins": 5154.5
},
{
"matches": 20669,
"model": "nemotron-4-340b-instruct",
"rank": 18,
"rating": 1063.9797181947,
"wins": 10567.5
},
{
"matches": 51739,
"model": "yi-large-preview",
"rank": 19,
"rating": 1060.8561460851,
"wins": 28793.0
},
{
"matches": 39631,
"model": "llama-2-70b-chat",
"rank": 20,
"rating": 1060.6350445555,
"wins": 18272.0
},
{
"matches": 6974,
"model": "reka-core-20240722",
"rank": 21,
"rating": 1059.9327626435,
"wins": 3490.0
},
{
"matches": 69418,
"model": "gemini-1.5-pro-api-0514",
"rank": 22,
"rating": 1059.8892020659,
"wins": 39302.5
},
{
"matches": 6565,
"model": "gemini-pro",
"rank": 23,
"rating": 1059.2020149016,
"wins": 3028.0
},
{
"matches": 18897,
"model": "llama-3.1-405b-instruct",
"rank": 24,
"rating": 1058.4003562569,
"wins": 10361.5
},
{
"matches": 11517,
"model": "mistral-large-2407",
"rank": 25,
"rating": 1058.1277726399,
"wins": 6038.5
},
{
"matches": 84505,
"model": "gpt-4-turbo-2024-04-09",
"rank": 26,
"rating": 1054.9014485987,
"wins": 48844.5
},
{
"matches": 92488,
"model": "gpt-4-1106-preview",
"rank": 27,
"rating": 1054.2069148591,
"wins": 57496.5
},
{
"matches": 38959,
"model": "gpt-3.5-turbo-0613",
"rank": 28,
"rating": 1052.6707187439,
"wins": 19308.0
},
{
"matches": 85867,
"model": "gpt-4-0125-preview",
"rank": 29,
"rating": 1050.1218586343,
"wins": 50802.5
},
{
"matches": 7579,
"model": "glm-4-0116",
"rank": 30,
"rating": 1049.1894798578,
"wins": 3566.0
},
{
"matches": 4858,
"model": "zephyr-orpo-141b-A35b-v0.1",
"rank": 31,
"rating": 1045.2216087838,
"wins": 2116.5
},
{
"matches": 25748,
"model": "qwen-max-0428",
"rank": 32,
"rating": 1044.8335252342,
"wins": 12269.5
},
{
"matches": 20633,
"model": "claude-instant-1",
"rank": 33,
"rating": 1044.7562986355,
"wins": 11178.0
},
{
"matches": 16672,
"model": "yi-large",
"rank": 34,
"rating": 1044.7374546104,
"wins": 8533.5
},
{
"matches": 3997,
"model": "deepseek-coder-v2-0724",
"rank": 35,
"rating": 1043.8935002553,
"wins": 1883.5
},
{
"matches": 16165,
"model": "deepseek-v2-api-0628",
"rank": 36,
"rating": 1042.4852039574,
"wins": 7994.5
},
{
"matches": 56129,
"model": "gemini-1.5-flash-api-0514",
"rank": 37,
"rating": 1041.3766839839,
"wins": 29778.0
},
{
"matches": 161827,
"model": "llama-3-70b-instruct",
"rank": 38,
"rating": 1040.8719399645,
"wins": 84977.0
},
{
"matches": 80911,
"model": "command-r-plus",
"rank": 39,
"rating": 1034.8822379837,
"wins": 40967.0
},
{
"matches": 55979,
"model": "gpt-4-0314",
"rank": 40,
"rating": 1033.6412751608,
"wins": 31098.0
},
{
"matches": 37702,
"model": "claude-2.1",
"rank": 41,
"rating": 1032.8143823524,
"wins": 16848.0
},
{
"matches": 8391,
"model": "wizardlm-70b",
"rank": 42,
"rating": 1032.5111769139,
"wins": 4325.0
},
{
"matches": 27838,
"model": "gemma-2-27b-it",
"rank": 43,
"rating": 1030.7976774989,
"wins": 13981.5
},
{
"matches": 1714,
"model": "dolphin-2.2.1-mistral-7b",
"rank": 44,
"rating": 1029.9575607586,
"wins": 738.5
},
{
"matches": 3003,
"model": "guanaco-33b",
"rank": 45,
"rating": 1027.2582838587,
"wins": 1552.0
},
{
"matches": 3843,
"model": "nous-hermes-2-mixtral-8x7b-dpo",
"rank": 46,
"rating": 1025.2145335894,
"wins": 1697.5
},
{
"matches": 7192,
"model": "wizardlm-13b",
"rank": 47,
"rating": 1024.793903105,
"wins": 3690.0
},
{
"matches": 2650,
"model": "mpt-30b-chat",
"rank": 48,
"rating": 1024.0563241095,
"wins": 1351.5
},
{
"matches": 27508,
"model": "qwen1.5-110b-chat",
"rank": 49,
"rating": 1023.7761579902,
"wins": 12612.0
},
{
"matches": 113106,
"model": "claude-3-sonnet-20240229",
"rank": 50,
"rating": 1023.6338810846,
"wins": 60284.5
},
{
"matches": 12377,
"model": "mistral-next",
"rank": 51,
"rating": 1021.9755255724,
"wins": 5952.5
},
{
"matches": 15801,
"model": "deepseek-coder-v2",
"rank": 52,
"rating": 1021.6778455715,
"wins": 7243.0
},
{
"matches": 16056,
"model": "reka-flash-21b-20240226-online",
"rank": 53,
"rating": 1020.8238893282,
"wins": 7554.5
},
{
"matches": 16667,
"model": "starling-lm-7b-beta",
"rank": 54,
"rating": 1020.1246920766,
"wins": 7626.0
},
{
"matches": 3636,
"model": "llama2-70b-steerlm-chat",
"rank": 55,
"rating": 1018.5882277478,
"wins": 1587.5
},
{
"matches": 35555,
"model": "mistral-medium",
"rank": 56,
"rating": 1018.0043978294,
"wins": 17731.5
},
{
"matches": 19740,
"model": "llama-2-13b-chat",
"rank": 57,
"rating": 1016.6652779795,
"wins": 8935.5
},
{
"matches": 6660,
"model": "tulu-2-dpo-70b",
"rank": 58,
"rating": 1014.6804751782,
"wins": 3231.0
},
{
"matches": 62688,
"model": "reka-core-20240501",
"rank": 59,
"rating": 1013.8222528866,
"wins": 31198.0
},
{
"matches": 89617,
"model": "gpt-4-0613",
"rank": 60,
"rating": 1013.3276942754,
"wins": 43878.0
},
{
"matches": 4981,
"model": "deepseek-llm-67b-chat",
"rank": 61,
"rating": 1011.6783077357,
"wins": 2017.0
},
{
"matches": 4291,
"model": "solar-10.7b-instruct-v1.0",
"rank": 62,
"rating": 1011.6055213958,
"wins": 1814.0
},
{
"matches": 12987,
"model": "openchat-3.5-0106",
"rank": 63,
"rating": 1009.5145969972,
"wins": 5974.0
},
{
"matches": 7156,
"model": "reka-flash-20240722",
"rank": 64,
"rating": 1003.5048766039,
"wins": 3290.0
},
{
"matches": 25248,
"model": "gemma-2-9b-it",
"rank": 65,
"rating": 1003.026164166,
"wins": 11661.5
},
{
"matches": 12671,
"model": "llama-3.1-8b-instruct",
"rank": 66,
"rating": 1002.2631876345,
"wins": 5121.0
},
{
"matches": 8117,
"model": "openchat-3.5",
"rank": 67,
"rating": 1001.939993772,
"wins": 3783.0
},
{
"matches": 6338,
"model": "pplx-7b-online",
"rank": 68,
"rating": 1001.6927968751,
"wins": 2572.0
},
{
"matches": 40643,
"model": "qwen1.5-72b-chat",
"rank": 69,
"rating": 1000.9425509065,
"wins": 19941.0
},
{
"matches": 1815,
"model": "zephyr-7b-alpha",
"rank": 70,
"rating": 999.8845726251,
"wins": 855.0
},
{
"matches": 109888,
"model": "claude-3-haiku-20240307",
"rank": 71,
"rating": 999.7072031054,
"wins": 55074.0
},
{
"matches": 10422,
"model": "starling-lm-7b-alpha",
"rank": 72,
"rating": 998.0936577805,
"wins": 4817.0
},
{
"matches": 25792,
"model": "reka-flash-21b-20240226",
"rank": 73,
"rating": 992.8107482264,
"wins": 11891.0
},
{
"matches": 64938,
"model": "mistral-large-2402",
"rank": 74,
"rating": 992.5035561283,
"wins": 30754.5
},
{
"matches": 17026,
"model": "gpt-3.5-turbo-1106",
"rank": 75,
"rating": 990.5828502496,
"wins": 7026.0
},
{
"matches": 4864,
"model": "qwen1.5-7b-chat",
"rank": 76,
"rating": 990.5290268684,
"wins": 1903.5
},
{
"matches": 20477,
"model": "reka-flash-preview-20240611",
"rank": 77,
"rating": 990.4289024537,
"wins": 9003.5
},
{
"matches": 25156,
"model": "yi-1.5-34b-chat",
"rank": 78,
"rating": 988.8079557694,
"wins": 11066.0
},
{
"matches": 5091,
"model": "openhermes-2.5-mistral-7b",
"rank": 79,
"rating": 988.507956639,
"wins": 2379.0
},
{
"matches": 7508,
"model": "codellama-34b-instruct",
"rank": 80,
"rating": 986.9963372601,
"wins": 3192.5
},
{
"matches": 18666,
"model": "qwen1.5-14b-chat",
"rank": 81,
"rating": 985.79676379,
"wins": 8265.0
},
{
"matches": 15946,
"model": "yi-34b-chat",
"rank": 82,
"rating": 984.9789753527,
"wins": 7423.5
},
{
"matches": 6890,
"model": "pplx-70b-online",
"rank": 83,
"rating": 984.0082457269,
"wins": 3118.0
},
{
"matches": 34410,
"model": "qwen2-72b-instruct",
"rank": 84,
"rating": 983.7228839439,
"wins": 16578.5
},
{
"matches": 33736,
"model": "dbrx-instruct-preview",
"rank": 85,
"rating": 983.0655232047,
"wins": 14350.5
},
{
"matches": 107100,
"model": "llama-3-8b-instruct",
"rank": 86,
"rating": 978.2506897907,
"wins": 49020.5
},
{
"matches": 1325,
"model": "falcon-180b-chat",
"rank": 87,
"rating": 976.0053410687,
"wins": 580.0
},
{
"matches": 8745,
"model": "palm-2",
"rank": 88,
"rating": 975.4724123669,
"wins": 3851.5
},
{
"matches": 5070,
"model": "qwen-14b-chat",
"rank": 89,
"rating": 974.4393986925,
"wins": 2262.5
},
{
"matches": 5270,
"model": "stripedhyena-nous-7b",
"rank": 90,
"rating": 972.5214538963,
"wins": 2001.0
},
{
"matches": 22783,
"model": "qwen1.5-32b-chat",
"rank": 91,
"rating": 970.5573357551,
"wins": 10038.5
},
{
"matches": 56373,
"model": "command-r",
"rank": 92,
"rating": 968.3429828775,
"wins": 25830.5
},
{
"matches": 9178,
"model": "gemma-7b-it",
"rank": 93,
"rating": 966.2526965022,
"wins": 3447.5
},
{
"matches": 11330,
"model": "zephyr-7b-beta",
"rank": 94,
"rating": 966.1969243012,
"wins": 5268.0
},
{
"matches": 50256,
"model": "mixtral-8x22b-instruct-v0.1",
"rank": 95,
"rating": 964.4141739849,
"wins": 22236.0
},
{
"matches": 7022,
"model": "vicuna-7b",
"rank": 96,
"rating": 963.6362226151,
"wins": 3187.5
},
{
"matches": 34197,
"model": "snowflake-arctic-instruct",
"rank": 97,
"rating": 961.9084215198,
"wins": 13326.5
},
{
"matches": 22938,
"model": "vicuna-33b",
"rank": 98,
"rating": 959.3823284501,
"wins": 10924.5
},
{
"matches": 10509,
"model": "gemma-2-2b-it",
"rank": 99,
"rating": 959.2713799268,
"wins": 3731.5
},
{
"matches": 7038,
"model": "koala-13b",
"rank": 100,
"rating": 954.395027749,
"wins": 3380.0
},
{
"matches": 68914,
"model": "gpt-3.5-turbo-0125",
"rank": 101,
"rating": 951.9181205239,
"wins": 29165.5
},
{
"matches": 9145,
"model": "mistral-7b-instruct",
"rank": 102,
"rating": 948.891187234,
"wins": 3685.5
},
{
"matches": 14559,
"model": "llama-2-7b-chat",
"rank": 103,
"rating": 948.4212635676,
"wins": 6212.0
},
{
"matches": 20074,
"model": "mistral-7b-instruct-v0.2",
"rank": 104,
"rating": 945.8420359878,
"wins": 8282.5
},
{
"matches": 25095,
"model": "gemma-1.1-7b-it",
"rank": 105,
"rating": 941.8141470663,
"wins": 9687.0
},
{
"matches": 1787,
"model": "gpt4all-13b-snoozy",
"rank": 106,
"rating": 938.6958755482,
"wins": 680.0
},
{
"matches": 18505,
"model": "phi-3-small-8k-instruct",
"rank": 107,
"rating": 930.6944270029,
"wins": 7100.0
},
{
"matches": 6496,
"model": "olmo-7b-instruct",
"rank": 108,
"rating": 928.8306135323,
"wins": 2157.0
},
{
"matches": 21129,
"model": "phi-3-mini-4k-instruct",
"rank": 109,
"rating": 928.4371869598,
"wins": 7094.0
},
{
"matches": 21617,
"model": "phi-3-mini-128k-instruct",
"rank": 110,
"rating": 928.3336295806,
"wins": 7094.5
},
{
"matches": 4938,
"model": "RWKV-4-Raven-14B",
"rank": 111,
"rating": 927.486785137,
"wins": 1965.5
},
{
"matches": 19794,
"model": "vicuna-13b",
"rank": 112,
"rating": 927.1632498042,
"wins": 9553.5
},
{
"matches": 1193,
"model": "codellama-70b-instruct",
"rank": 113,
"rating": 926.7141337337,
"wins": 411.5
},
{
"matches": 4018,
"model": "mpt-7b-chat",
"rank": 114,
"rating": 922.0268289601,
"wins": 1573.5
},
{
"matches": 76064,
"model": "mixtral-8x7b-instruct-v0.1",
"rank": 115,
"rating": 920.8542542685,
"wins": 33437.0
},
{
"matches": 22427,
"model": "phi-3-medium-4k-instruct",
"rank": 116,
"rating": 919.7045738753,
"wins": 8854.0
},
{
"matches": 4921,
"model": "gemma-2b-it",
"rank": 117,
"rating": 918.581510353,
"wins": 1513.0
},
{
"matches": 10484,
"model": "phi-3-mini-4k-instruct-june-2024",
"rank": 118,
"rating": 913.3885931532,
"wins": 3097.5
},
{
"matches": 7811,
"model": "qwen1.5-4b-chat",
"rank": 119,
"rating": 910.8121423587,
"wins": 2364.5
},
{
"matches": 4995,
"model": "chatglm-6b",
"rank": 120,
"rating": 908.8611775605,
"wins": 1834.5
},
{
"matches": 5874,
"model": "alpaca-13b",
"rank": 121,
"rating": 904.2619553966,
"wins": 2330.0
},
{
"matches": 11363,
"model": "gemma-1.1-2b-it",
"rank": 122,
"rating": 892.2955737397,
"wins": 3282.0
},
{
"matches": 2707,
"model": "chatglm2-6b",
"rank": 123,
"rating": 888.2580854124,
"wins": 808.0
},
{
"matches": 3334,
"model": "stablelm-tuned-alpha-7b",
"rank": 124,
"rating": 882.7115402984,
"wins": 1141.5
},
{
"matches": 4763,
"model": "chatglm3-6b",
"rank": 125,
"rating": 867.498613261,
"wins": 1454.0
},
{
"matches": 6382,
"model": "oasst-pythia-12b",
"rank": 126,
"rating": 865.3424632797,
"wins": 2425.0
},
{
"matches": 2443,
"model": "llama-13b",
"rank": 127,
"rating": 852.9506118392,
"wins": 745.5
},
{
"matches": 4304,
"model": "fastchat-t5-3b",
"rank": 128,
"rating": 847.3254654092,
"wins": 1465.0
},
{
"matches": 3484,
"model": "dolly-v2-12b",
"rank": 129,
"rating": 834.3876685486,
"wins": 1099.5
}
]
chapter7/elo-leaderboard/validation/runs/exp7-7-arena-20260731-v1/rating_history.json
+17546
View File
File diff suppressed because it is too large Load Diff
chapter7/elo-leaderboard/validation/runs/exp7-7-arena-20260731-v1/rating_history.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 189 KiB

chapter7/elo-leaderboard/validation/runs/exp7-7-arena-20260731-v1/summary.json
+387
View File
@@ -0,0 +1,387 @@
{
"dataset": {
"accepted_records": 1670250,
"bounded_test_run": false,
"bytes": 2131976365,
"end_utc": "2024-08-14T19:52:53.409200+00:00",
"load_filter_seconds": 15.133,
"model_count": 129,
"outcomes": {
"model_a": 543065,
"model_b": 550810,
"tie": 281121,
"tie (bothbad)": 295254
},
"path_recorded_as": "arena_data.json",
"sha256": "747c1c937dfa941d5a455a1fd70e2879d29642058da39b97996b977233b9bd1b",
"source_records": 1799991,
"start_utc": "2023-04-24T15:53:11.132200+00:00",
"url": "https://storage.googleapis.com/arena_external_data/public/clean_battle_20240814_public.json"
},
"experiment": "7-7",
"gates": {
"bradley_terry_official_method_completed": true,
"chronological_online_elo_k4_completed": true,
"d3_animation_saved": true,
"millions_of_blind_votes_loaded": true,
"monthly_history_saved": true,
"official_public_arena_snapshot_hashed": true,
"online_vs_official_method_rank_agreement_observed": true,
"pairwise_empirical_and_predicted_matrix_saved": true,
"static_visualizations_saved": true
},
"generated_at_utc": "2026-07-31T09:59:17.815099+00:00",
"official_complete": true,
"protocol": {
"bootstrap_random_seed": 0,
"bootstrap_rounds": 20,
"history_interval": "monthly cumulative snapshots",
"official_method": "Bradley-Terry maximum-likelihood reconstruction",
"online_elo": "initial=1000, K=4, stable chronological order"
},
"results": {
"official_method_top_20": [
{
"lower_ci": 1199.4356201692,
"model": "chatgpt-4o-latest",
"rank": 1,
"rating": 1202.8722324768,
"upper_ci": 1207.3805726125
},
{
"lower_ci": 1182.767114327,
"model": "gemini-1.5-pro-exp-0801",
"rank": 2,
"rating": 1187.3727762743,
"upper_ci": 1191.7564879767
},
{
"lower_ci": 1173.1270556478,
"model": "gpt-4o-2024-05-13",
"rank": 3,
"rating": 1174.5475812185,
"upper_ci": 1177.0403252386
},
{
"lower_ci": 1161.1978031325,
"model": "gpt-4o-mini-2024-07-18",
"rank": 4,
"rating": 1162.9712796813,
"upper_ci": 1166.568334134
},
{
"lower_ci": 1156.0224735749,
"model": "claude-3-5-sonnet-20240620",
"rank": 5,
"rating": 1159.6692177384,
"upper_ci": 1161.8458744744
},
{
"lower_ci": 1153.6447817793,
"model": "gemini-advanced-0514",
"rank": 6,
"rating": 1155.6305094344,
"upper_ci": 1157.1030226933
},
{
"lower_ci": 1150.4614200682,
"model": "llama-3.1-405b-instruct",
"rank": 7,
"rating": 1153.3950109278,
"upper_ci": 1157.0848375454
},
{
"lower_ci": 1146.348878889,
"model": "gpt-4o-2024-08-06",
"rank": 8,
"rating": 1150.7428051987,
"upper_ci": 1156.3245456986
},
{
"lower_ci": 1146.7772301944,
"model": "gemini-1.5-pro-api-0514",
"rank": 9,
"rating": 1149.1265569601,
"upper_ci": 1151.5010539806
},
{
"lower_ci": 1143.6682527667,
"model": "gemini-1.5-pro-api-0409-preview",
"rank": 10,
"rating": 1146.1756037638,
"upper_ci": 1147.8895040945
},
{
"lower_ci": 1142.3990049162,
"model": "gpt-4-turbo-2024-04-09",
"rank": 11,
"rating": 1144.9187340632,
"upper_ci": 1146.8362809167
},
{
"lower_ci": 1138.1303324947,
"model": "gpt-4-1106-preview",
"rank": 12,
"rating": 1139.5004505155,
"upper_ci": 1140.97644635
},
{
"lower_ci": 1136.3852091186,
"model": "mistral-large-2407",
"rank": 13,
"rating": 1139.3566231011,
"upper_ci": 1143.8960727747
},
{
"lower_ci": 1135.0624903381,
"model": "athene-70b-0725",
"rank": 14,
"rating": 1138.2016223593,
"upper_ci": 1141.397916447
},
{
"lower_ci": 1135.0278139027,
"model": "claude-3-opus-20240229",
"rank": 15,
"rating": 1136.5739464324,
"upper_ci": 1137.5663039742
},
{
"lower_ci": 1131.7219814044,
"model": "llama-3.1-70b-instruct",
"rank": 16,
"rating": 1134.8942150397,
"upper_ci": 1139.3028524197
},
{
"lower_ci": 1132.1800230236,
"model": "gpt-4-0125-preview",
"rank": 17,
"rating": 1133.974390087,
"upper_ci": 1135.7552442263
},
{
"lower_ci": 1127.3300289872,
"model": "yi-large-preview",
"rank": 18,
"rating": 1128.1692229165,
"upper_ci": 1131.371743138
},
{
"lower_ci": 1111.850158929,
"model": "reka-core-20240722",
"rank": 19,
"rating": 1117.1464671073,
"upper_ci": 1121.9537941313
},
{
"lower_ci": 1114.0244965884,
"model": "gemini-1.5-flash-api-0514",
"rank": 20,
"rating": 1116.6161359294,
"upper_ci": 1118.5223962538
}
],
"online_top_20": [
{
"matches": 55654,
"model": "gemini-1.5-pro-api-0409-preview",
"rank": 1,
"rating": 1159.8203895968,
"wins": 33738.5
},
{
"matches": 20071,
"model": "gemini-1.5-pro-exp-0801",
"rank": 2,
"rating": 1127.8877886511,
"wins": 12139.5
},
{
"matches": 14514,
"model": "chatgpt-4o-latest",
"rank": 3,
"rating": 1126.6935165964,
"wins": 8999.0
},
{
"matches": 5655,
"model": "gpt-3.5-turbo-0314",
"rank": 4,
"rating": 1103.9751202512,
"wins": 3789.0
},
{
"matches": 11827,
"model": "bard-jan-24-gemini-pro",
"rank": 5,
"rating": 1094.2118678517,
"wins": 7075.5
},
{
"matches": 21172,
"model": "claude-1",
"rank": 6,
"rating": 1092.7309868534,
"wins": 12081.5
},
{
"matches": 52155,
"model": "gemini-advanced-0514",
"rank": 7,
"rating": 1087.5408038371,
"wins": 30523.0
},
{
"matches": 13604,
"model": "llama-3.1-70b-instruct",
"rank": 8,
"rating": 1087.3529845908,
"wins": 7058.5
},
{
"matches": 77509,
"model": "gpt-4o-2024-05-13",
"rank": 9,
"rating": 1086.7242770205,
"wins": 46951.0
},
{
"matches": 9761,
"model": "gpt-4o-2024-08-06",
"rank": 10,
"rating": 1085.601246973,
"wins": 5323.0
},
{
"matches": 19370,
"model": "gpt-4o-mini-2024-07-18",
"rank": 11,
"rating": 1079.187026245,
"wins": 11082.0
},
{
"matches": 47703,
"model": "claude-3-5-sonnet-20240620",
"rank": 12,
"rating": 1079.0096726066,
"wins": 27107.5
},
{
"matches": 155944,
"model": "claude-3-opus-20240229",
"rank": 13,
"rating": 1074.9078256001,
"wins": 91432.5
},
{
"matches": 11476,
"model": "athene-70b-0725",
"rank": 14,
"rating": 1073.7607622933,
"wins": 6098.0
},
{
"matches": 18790,
"model": "gemini-pro-dev-api",
"rank": 15,
"rating": 1072.3072263884,
"wins": 8728.5
},
{
"matches": 12774,
"model": "claude-2.0",
"rank": 16,
"rating": 1069.530762176,
"wins": 6814.5
},
{
"matches": 10239,
"model": "glm-4-0520",
"rank": 17,
"rating": 1065.2967580553,
"wins": 5154.5
},
{
"matches": 20669,
"model": "nemotron-4-340b-instruct",
"rank": 18,
"rating": 1063.9797181947,
"wins": 10567.5
},
{
"matches": 51739,
"model": "yi-large-preview",
"rank": 19,
"rating": 1060.8561460851,
"wins": 28793.0
},
{
"matches": 39631,
"model": "llama-2-70b-chat",
"rank": 20,
"rating": 1060.6350445555,
"wins": 18272.0
}
],
"rank_comparison": {
"claim_boundary": "This is a same-snapshot reconstruction of the official method, not a scrape of the mutable live leaderboard. Scores need not match the live service.",
"common_models": 129,
"comparison_target": "Bradley-Terry MLE reconstruction used by Chatbot Arena",
"kendall_rank_correlation": 0.606347,
"official_method_top_20": [
"chatgpt-4o-latest",
"gemini-1.5-pro-exp-0801",
"gpt-4o-2024-05-13",
"gpt-4o-mini-2024-07-18",
"claude-3-5-sonnet-20240620",
"gemini-advanced-0514",
"llama-3.1-405b-instruct",
"gpt-4o-2024-08-06",
"gemini-1.5-pro-api-0514",
"gemini-1.5-pro-api-0409-preview",
"gpt-4-turbo-2024-04-09",
"gpt-4-1106-preview",
"mistral-large-2407",
"athene-70b-0725",
"claude-3-opus-20240229",
"llama-3.1-70b-instruct",
"gpt-4-0125-preview",
"yi-large-preview",
"reka-core-20240722",
"gemini-1.5-flash-api-0514"
],
"online_top_20": [
"gemini-1.5-pro-api-0409-preview",
"gemini-1.5-pro-exp-0801",
"chatgpt-4o-latest",
"gpt-3.5-turbo-0314",
"bard-jan-24-gemini-pro",
"claude-1",
"gemini-advanced-0514",
"llama-3.1-70b-instruct",
"gpt-4o-2024-05-13",
"gpt-4o-2024-08-06",
"gpt-4o-mini-2024-07-18",
"claude-3-5-sonnet-20240620",
"claude-3-opus-20240229",
"athene-70b-0725",
"gemini-pro-dev-api",
"claude-2.0",
"glm-4-0520",
"nemotron-4-340b-instruct",
"yi-large-preview",
"llama-2-70b-chat"
],
"spearman_rank_correlation": 0.786717,
"top_20_overlap": 12
}
},
"schema_version": 1,
"status": "passed",
"timing_seconds": {
"bradley_terry": 71.472,
"online_and_history": 8.301,
"total": 97.248
}
}
chapter7/elo-leaderboard/validation/runs/exp7-7-arena-20260731-v1/win_rate_matrix.json
+908
View File
@@ -0,0 +1,908 @@
{
"empirical": {
"athene-70b-0725": {
"athene-70b-0725": 0.5,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": 0.41589648798521256,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.49077181208053694,
"claude-3-opus-20240229": 0.5027472527472527,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": 0.43097014925373134,
"gemini-advanced-0514": 0.48249027237354086,
"gemini-pro-dev-api": NaN,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.4431239388794567,
"gpt-4o-2024-08-06": 0.505249343832021,
"gpt-4o-mini-2024-07-18": 0.4753787878787879,
"llama-2-70b-chat": NaN,
"llama-3.1-70b-instruct": 0.5317460317460317,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": 0.5294117647058824
},
"bard-jan-24-gemini-pro": {
"athene-70b-0725": NaN,
"bard-jan-24-gemini-pro": 0.5,
"chatgpt-4o-latest": NaN,
"claude-1": 0.572289156626506,
"claude-2.0": 0.6696428571428571,
"claude-3-5-sonnet-20240620": NaN,
"claude-3-opus-20240229": 0.3,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": NaN,
"gemini-advanced-0514": NaN,
"gemini-pro-dev-api": 0.6332794830371568,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": NaN,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": NaN,
"llama-2-70b-chat": 0.6492537313432836,
"llama-3.1-70b-instruct": NaN,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": NaN
},
"chatgpt-4o-latest": {
"athene-70b-0725": 0.5841035120147874,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": 0.5,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.5534351145038168,
"claude-3-opus-20240229": 0.5915750915750916,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": 0.5329736211031175,
"gemini-advanced-0514": NaN,
"gemini-pro-dev-api": NaN,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.5442092154420921,
"gpt-4o-2024-08-06": 0.5164835164835165,
"gpt-4o-mini-2024-07-18": 0.6010733452593918,
"llama-2-70b-chat": NaN,
"llama-3.1-70b-instruct": 0.5793838862559242,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": NaN
},
"claude-1": {
"athene-70b-0725": NaN,
"bard-jan-24-gemini-pro": 0.42771084337349397,
"chatgpt-4o-latest": NaN,
"claude-1": 0.5,
"claude-2.0": 0.5460750853242321,
"claude-3-5-sonnet-20240620": NaN,
"claude-3-opus-20240229": 0.34452296819787986,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": NaN,
"gemini-advanced-0514": NaN,
"gemini-pro-dev-api": 0.5342465753424658,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": 0.5548780487804879,
"gpt-4o-2024-05-13": NaN,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": NaN,
"llama-2-70b-chat": 0.604,
"llama-3.1-70b-instruct": NaN,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": NaN
},
"claude-2.0": {
"athene-70b-0725": NaN,
"bard-jan-24-gemini-pro": 0.33035714285714285,
"chatgpt-4o-latest": NaN,
"claude-1": 0.4539249146757679,
"claude-2.0": 0.5,
"claude-3-5-sonnet-20240620": NaN,
"claude-3-opus-20240229": NaN,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": NaN,
"gemini-advanced-0514": NaN,
"gemini-pro-dev-api": 0.5569620253164557,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": NaN,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": NaN,
"llama-2-70b-chat": 0.5866834170854272,
"llama-3.1-70b-instruct": NaN,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": NaN
},
"claude-3-5-sonnet-20240620": {
"athene-70b-0725": 0.5092281879194631,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": 0.44656488549618323,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.5,
"claude-3-opus-20240229": 0.5447761194029851,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": 0.4537037037037037,
"gemini-advanced-0514": 0.49312714776632305,
"gemini-pro-dev-api": NaN,
"glm-4-0520": 0.5800756620428752,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.47875730217737655,
"gpt-4o-2024-08-06": 0.5189873417721519,
"gpt-4o-mini-2024-07-18": 0.47484909456740443,
"llama-2-70b-chat": NaN,
"llama-3.1-70b-instruct": 0.538961038961039,
"nemotron-4-340b-instruct": 0.589835728952772,
"yi-large-preview": 0.5205761316872428
},
"claude-3-opus-20240229": {
"athene-70b-0725": 0.49725274725274726,
"bard-jan-24-gemini-pro": 0.7,
"chatgpt-4o-latest": 0.4084249084249084,
"claude-1": 0.6554770318021201,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.4552238805970149,
"claude-3-opus-20240229": 0.5,
"gemini-1.5-pro-api-0409-preview": 0.4815616343490305,
"gemini-1.5-pro-exp-0801": 0.4305157593123209,
"gemini-advanced-0514": 0.46121231155778897,
"gemini-pro-dev-api": 0.6489533011272142,
"glm-4-0520": 0.5424311926605505,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.4481694338016177,
"gpt-4o-2024-08-06": 0.4369627507163324,
"gpt-4o-mini-2024-07-18": 0.4437386569872958,
"llama-2-70b-chat": 0.6890951276102089,
"llama-3.1-70b-instruct": 0.4813519813519814,
"nemotron-4-340b-instruct": 0.5431309904153354,
"yi-large-preview": 0.5238190552441954
},
"gemini-1.5-pro-api-0409-preview": {
"athene-70b-0725": NaN,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": NaN,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": NaN,
"claude-3-opus-20240229": 0.5184383656509696,
"gemini-1.5-pro-api-0409-preview": 0.5,
"gemini-1.5-pro-exp-0801": NaN,
"gemini-advanced-0514": NaN,
"gemini-pro-dev-api": NaN,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": NaN,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": NaN,
"llama-2-70b-chat": 0.6875,
"llama-3.1-70b-instruct": NaN,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": NaN
},
"gemini-1.5-pro-exp-0801": {
"athene-70b-0725": 0.5690298507462687,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": 0.4670263788968825,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.5462962962962963,
"claude-3-opus-20240229": 0.5694842406876791,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": 0.5,
"gemini-advanced-0514": 0.5263157894736842,
"gemini-pro-dev-api": NaN,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.5229445506692161,
"gpt-4o-2024-08-06": 0.5241071428571429,
"gpt-4o-mini-2024-07-18": 0.529559748427673,
"llama-2-70b-chat": NaN,
"llama-3.1-70b-instruct": 0.5773092369477911,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": 0.5862068965517241
},
"gemini-advanced-0514": {
"athene-70b-0725": 0.5175097276264592,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": NaN,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.506872852233677,
"claude-3-opus-20240229": 0.538787688442211,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": 0.47368421052631576,
"gemini-advanced-0514": 0.5,
"gemini-pro-dev-api": NaN,
"glm-4-0520": 0.5773381294964028,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.4901780233271946,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": 0.4604072398190045,
"llama-2-70b-chat": 0.6145833333333334,
"llama-3.1-70b-instruct": 0.5423076923076923,
"nemotron-4-340b-instruct": 0.5807407407407408,
"yi-large-preview": 0.5602636534839924
},
"gemini-pro-dev-api": {
"athene-70b-0725": NaN,
"bard-jan-24-gemini-pro": 0.3667205169628433,
"chatgpt-4o-latest": NaN,
"claude-1": 0.4657534246575342,
"claude-2.0": 0.4430379746835443,
"claude-3-5-sonnet-20240620": NaN,
"claude-3-opus-20240229": 0.35104669887278583,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": NaN,
"gemini-advanced-0514": NaN,
"gemini-pro-dev-api": 0.5,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": NaN,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": NaN,
"llama-2-70b-chat": 0.5535714285714286,
"llama-3.1-70b-instruct": NaN,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": NaN
},
"glm-4-0520": {
"athene-70b-0725": NaN,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": NaN,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.41992433795712486,
"claude-3-opus-20240229": 0.45756880733944955,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": NaN,
"gemini-advanced-0514": 0.4226618705035971,
"gemini-pro-dev-api": NaN,
"glm-4-0520": 0.5,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.3613861386138614,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": NaN,
"llama-2-70b-chat": NaN,
"llama-3.1-70b-instruct": NaN,
"nemotron-4-340b-instruct": 0.5087025316455697,
"yi-large-preview": 0.448943661971831
},
"gpt-3.5-turbo-0314": {
"athene-70b-0725": NaN,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": NaN,
"claude-1": 0.4451219512195122,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": NaN,
"claude-3-opus-20240229": NaN,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": NaN,
"gemini-advanced-0514": NaN,
"gemini-pro-dev-api": NaN,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": 0.5,
"gpt-4o-2024-05-13": NaN,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": NaN,
"llama-2-70b-chat": NaN,
"llama-3.1-70b-instruct": NaN,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": NaN
},
"gpt-4o-2024-05-13": {
"athene-70b-0725": 0.5568760611205433,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": 0.45579078455790784,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.5212426978226234,
"claude-3-opus-20240229": 0.5518305661983823,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": 0.4770554493307839,
"gemini-advanced-0514": 0.5098219766728054,
"gemini-pro-dev-api": NaN,
"glm-4-0520": 0.6386138613861386,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.5,
"gpt-4o-2024-08-06": 0.544973544973545,
"gpt-4o-mini-2024-07-18": 0.5163472378804961,
"llama-2-70b-chat": 0.5607476635514018,
"llama-3.1-70b-instruct": 0.5650224215246636,
"nemotron-4-340b-instruct": 0.5968559837728195,
"yi-large-preview": 0.5756445047489823
},
"gpt-4o-2024-08-06": {
"athene-70b-0725": 0.494750656167979,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": 0.4835164835164835,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.4810126582278481,
"claude-3-opus-20240229": 0.5630372492836676,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": 0.4758928571428571,
"gemini-advanced-0514": NaN,
"gemini-pro-dev-api": NaN,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.455026455026455,
"gpt-4o-2024-08-06": 0.5,
"gpt-4o-mini-2024-07-18": 0.5052910052910053,
"llama-2-70b-chat": NaN,
"llama-3.1-70b-instruct": 0.5273037542662116,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": NaN
},
"gpt-4o-mini-2024-07-18": {
"athene-70b-0725": 0.5246212121212122,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": 0.39892665474060823,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.5251509054325956,
"claude-3-opus-20240229": 0.5562613430127041,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": 0.47044025157232705,
"gemini-advanced-0514": 0.5395927601809954,
"gemini-pro-dev-api": NaN,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.4836527621195039,
"gpt-4o-2024-08-06": 0.4947089947089947,
"gpt-4o-mini-2024-07-18": 0.5,
"llama-2-70b-chat": NaN,
"llama-3.1-70b-instruct": 0.5584756898817346,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": 0.553125
},
"llama-2-70b-chat": {
"athene-70b-0725": NaN,
"bard-jan-24-gemini-pro": 0.35074626865671643,
"chatgpt-4o-latest": NaN,
"claude-1": 0.396,
"claude-2.0": 0.41331658291457285,
"claude-3-5-sonnet-20240620": NaN,
"claude-3-opus-20240229": 0.3109048723897912,
"gemini-1.5-pro-api-0409-preview": 0.3125,
"gemini-1.5-pro-exp-0801": NaN,
"gemini-advanced-0514": 0.3854166666666667,
"gemini-pro-dev-api": 0.44642857142857145,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.4392523364485981,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": NaN,
"llama-2-70b-chat": 0.5,
"llama-3.1-70b-instruct": NaN,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": 0.42142857142857143
},
"llama-3.1-70b-instruct": {
"athene-70b-0725": 0.46825396825396826,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": 0.4206161137440758,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.461038961038961,
"claude-3-opus-20240229": 0.5186480186480187,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": 0.4226907630522088,
"gemini-advanced-0514": 0.4576923076923077,
"gemini-pro-dev-api": NaN,
"glm-4-0520": NaN,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.4349775784753363,
"gpt-4o-2024-08-06": 0.4726962457337884,
"gpt-4o-mini-2024-07-18": 0.44152431011826543,
"llama-2-70b-chat": NaN,
"llama-3.1-70b-instruct": 0.5,
"nemotron-4-340b-instruct": NaN,
"yi-large-preview": 0.58
},
"nemotron-4-340b-instruct": {
"athene-70b-0725": NaN,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": NaN,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.4101642710472279,
"claude-3-opus-20240229": 0.45686900958466453,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": NaN,
"gemini-advanced-0514": 0.4192592592592593,
"gemini-pro-dev-api": NaN,
"glm-4-0520": 0.4912974683544304,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.40314401622718055,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": NaN,
"llama-2-70b-chat": NaN,
"llama-3.1-70b-instruct": NaN,
"nemotron-4-340b-instruct": 0.5,
"yi-large-preview": 0.44568690095846647
},
"yi-large-preview": {
"athene-70b-0725": 0.47058823529411764,
"bard-jan-24-gemini-pro": NaN,
"chatgpt-4o-latest": NaN,
"claude-1": NaN,
"claude-2.0": NaN,
"claude-3-5-sonnet-20240620": 0.4794238683127572,
"claude-3-opus-20240229": 0.47618094475580464,
"gemini-1.5-pro-api-0409-preview": NaN,
"gemini-1.5-pro-exp-0801": 0.41379310344827586,
"gemini-advanced-0514": 0.4397363465160075,
"gemini-pro-dev-api": NaN,
"glm-4-0520": 0.551056338028169,
"gpt-3.5-turbo-0314": NaN,
"gpt-4o-2024-05-13": 0.4243554952510176,
"gpt-4o-2024-08-06": NaN,
"gpt-4o-mini-2024-07-18": 0.446875,
"llama-2-70b-chat": 0.5785714285714286,
"llama-3.1-70b-instruct": 0.42,
"nemotron-4-340b-instruct": 0.5543130990415336,
"yi-large-preview": 0.5
}
},
"models": [
"gemini-1.5-pro-api-0409-preview",
"gemini-1.5-pro-exp-0801",
"chatgpt-4o-latest",
"gpt-3.5-turbo-0314",
"bard-jan-24-gemini-pro",
"claude-1",
"gemini-advanced-0514",
"llama-3.1-70b-instruct",
"gpt-4o-2024-05-13",
"gpt-4o-2024-08-06",
"gpt-4o-mini-2024-07-18",
"claude-3-5-sonnet-20240620",
"claude-3-opus-20240229",
"athene-70b-0725",
"gemini-pro-dev-api",
"claude-2.0",
"glm-4-0520",
"nemotron-4-340b-instruct",
"yi-large-preview",
"llama-2-70b-chat"
],
"online_elo_predicted": {
"athene-70b-0725": {
"athene-70b-0725": 0.5,
"bard-jan-24-gemini-pro": 0.4706024381703526,
"chatgpt-4o-latest": 0.4244076072370654,
"claude-1": 0.4727267496028405,
"claude-2.0": 0.5060871587475949,
"claude-3-5-sonnet-20240620": 0.4924467854859755,
"claude-3-opus-20240229": 0.4983492492034077,
"gemini-1.5-pro-api-0409-preview": 0.3786225197416955,
"gemini-1.5-pro-exp-0801": 0.4227290744565508,
"gemini-advanced-0514": 0.48017931822260296,
"gemini-pro-dev-api": 0.5020917941129024,
"glm-4-0520": 0.5121782721655728,
"gpt-3.5-turbo-0314": 0.45662732701186726,
"gpt-4o-2024-05-13": 0.4813526553622729,
"gpt-4o-2024-08-06": 0.482966766553394,
"gpt-4o-mini-2024-07-18": 0.492191613318456,
"llama-2-70b-chat": 0.5188804447839834,
"llama-3.1-70b-instruct": 0.4804491928529648,
"nemotron-4-340b-instruct": 0.5140723365206418,
"yi-large-preview": 0.5185627002158419
},
"bard-jan-24-gemini-pro": {
"athene-70b-0725": 0.5293975618296474,
"bard-jan-24-gemini-pro": 0.5,
"chatgpt-4o-latest": 0.45339086380344346,
"claude-1": 0.5021311461637057,
"claude-2.0": 0.5354593391190725,
"claude-3-5-sonnet-20240620": 0.5218637663710245,
"claude-3-opus-20240229": 0.5277521980731233,
"gemini-1.5-pro-api-0409-preview": 0.4066882610443156,
"gemini-1.5-pro-exp-0801": 0.45168765548015544,
"gemini-advanced-0514": 0.5095992532128728,
"gemini-pro-dev-api": 0.5314816122659721,
"glm-4-0520": 0.54151638062692,
"gpt-3.5-turbo-0314": 0.48595324754769137,
"gpt-4o-2024-05-13": 0.5107738414883239,
"gpt-4o-2024-08-06": 0.5123891431197954,
"gpt-4o-mini-2024-07-18": 0.5216090163334262,
"llama-2-70b-chat": 0.5481710593389454,
"llama-3.1-70b-instruct": 0.5098694443797749,
"nemotron-4-340b-instruct": 0.5433980845222826,
"yi-large-preview": 0.5478558027577041
},
"chatgpt-4o-latest": {
"athene-70b-0725": 0.5755923927629346,
"bard-jan-24-gemini-pro": 0.5466091361965566,
"chatgpt-4o-latest": 0.5,
"claude-1": 0.5487209243907352,
"claude-2.0": 0.5815294906471586,
"claude-3-5-sonnet-20240620": 0.5681949260650095,
"claude-3-opus-20240229": 0.5739785673924995,
"gemini-1.5-pro-api-0409-preview": 0.45247054063006786,
"gemini-1.5-pro-exp-0801": 0.49828131112534013,
"gemini-advanced-0514": 0.5561079756808931,
"gemini-pro-dev-api": 0.5776350830826076,
"glm-4-0520": 0.587448649362509,
"gpt-3.5-turbo-0314": 0.5326478829339771,
"gpt-4o-2024-05-13": 0.5572679471226406,
"gpt-4o-2024-08-06": 0.5588623197754942,
"gpt-4o-mini-2024-07-18": 0.5679444241258642,
"llama-2-70b-chat": 0.5939365657179976,
"llama-3.1-70b-instruct": 0.5563748494378427,
"nemotron-4-340b-instruct": 0.5892848182429368,
"yi-large-preview": 0.5936295693344523
},
"claude-1": {
"athene-70b-0725": 0.5272732503971594,
"bard-jan-24-gemini-pro": 0.4978688538362943,
"chatgpt-4o-latest": 0.45127907560926495,
"claude-1": 0.5,
"claude-2.0": 0.5333382703189705,
"claude-3-5-sonnet-20240620": 0.5197362986493318,
"claude-3-opus-20240229": 0.5256271146687194,
"gemini-1.5-pro-api-0409-preview": 0.4046329739918908,
"gemini-1.5-pro-exp-0801": 0.44957727554595445,
"gemini-advanced-0514": 0.5074687182117393,
"gemini-pro-dev-api": 0.5293583449328568,
"glm-4-0520": 0.539399178222487,
"gpt-3.5-turbo-0314": 0.483824038337802,
"gpt-4o-2024-05-13": 0.5086434891644779,
"gpt-4o-2024-08-06": 0.5102590804411641,
"gpt-4o-mini-2024-07-18": 0.519481458808123,
"llama-2-70b-chat": 0.5460588266923004,
"llama-3.1-70b-instruct": 0.5077389493164274,
"nemotron-4-340b-instruct": 0.541282210739082,
"yi-large-preview": 0.5457433176189294
},
"claude-2.0": {
"athene-70b-0725": 0.4939128412524052,
"bard-jan-24-gemini-pro": 0.46454066088092755,
"chatgpt-4o-latest": 0.41847050935284147,
"claude-1": 0.46666172968102954,
"claude-2.0": 0.5,
"claude-3-5-sonnet-20240620": 0.4863621348844711,
"claude-3-opus-20240229": 0.4922624014572012,
"gemini-1.5-pro-api-0409-preview": 0.3729109568972296,
"gemini-1.5-pro-exp-0801": 0.41679845455453196,
"gemini-advanced-0514": 0.474104656737174,
"gemini-pro-dev-api": 0.4960044318617072,
"glm-4-0520": 0.5060929201168735,
"gpt-3.5-turbo-0314": 0.4505923459773481,
"gpt-4o-2024-05-13": 0.4752767219072891,
"gpt-4o-2024-08-06": 0.4768891926892078,
"gpt-4o-mini-2024-07-18": 0.4861070959392006,
"llama-2-70b-chat": 0.5127991699819673,
"llama-3.1-70b-instruct": 0.47437423287938824,
"nemotron-4-340b-instruct": 0.5079879147696204,
"yi-large-preview": 0.512481182668407
},
"claude-3-5-sonnet-20240620": {
"athene-70b-0725": 0.5075532145140246,
"bard-jan-24-gemini-pro": 0.4781362336289755,
"chatgpt-4o-latest": 0.4318050739349906,
"claude-1": 0.4802637013506682,
"claude-2.0": 0.5136378651155289,
"claude-3-5-sonnet-20240620": 0.5,
"claude-3-opus-20240229": 0.505902758110997,
"gemini-1.5-pro-api-0409-preview": 0.38575678604471353,
"gemini-1.5-pro-exp-0801": 0.4301191468425039,
"gemini-advanced-0514": 0.4877251820914822,
"gemini-pro-dev-api": 0.5096443991097894,
"glm-4-0520": 0.519724229338601,
"gpt-3.5-turbo-0314": 0.46413354167370846,
"gpt-4o-2024-05-13": 0.48889961603567494,
"gpt-4o-2024-08-06": 0.4905150999228697,
"gpt-4o-mini-2024-07-18": 0.49974476761967085,
"llama-2-70b-chat": 0.526418589284047,
"llama-3.1-70b-instruct": 0.487995316371302,
"nemotron-4-340b-instruct": 0.5216163605038188,
"yi-large-preview": 0.5261012762929619
},
"claude-3-opus-20240229": {
"athene-70b-0725": 0.5016507507965924,
"bard-jan-24-gemini-pro": 0.47224780192687665,
"chatgpt-4o-latest": 0.4260214326075004,
"claude-1": 0.47437288533128064,
"claude-2.0": 0.5077375985427988,
"claude-3-5-sonnet-20240620": 0.49409724188900295,
"claude-3-opus-20240229": 0.5,
"gemini-1.5-pro-api-0409-preview": 0.3801772378799626,
"gemini-1.5-pro-exp-0801": 0.4243412226190692,
"gemini-advanced-0514": 0.4818276906995902,
"gemini-pro-dev-api": 0.5037424932177174,
"glm-4-0520": 0.513827911016007,
"gpt-3.5-turbo-0314": 0.4582661256484433,
"gpt-4o-2024-05-13": 0.4830013131364567,
"gpt-4o-2024-08-06": 0.48461578707994823,
"gpt-4o-mini-2024-07-18": 0.4938420466183484,
"llama-2-70b-chat": 0.520528636323993,
"llama-3.1-70b-instruct": 0.48209763256058513,
"nemotron-4-340b-instruct": 0.5157216264686881,
"yi-large-preview": 0.5202109737620978
},
"gemini-1.5-pro-api-0409-preview": {
"athene-70b-0725": 0.6213774802583045,
"bard-jan-24-gemini-pro": 0.5933117389556843,
"chatgpt-4o-latest": 0.5475294593699321,
"claude-1": 0.5953670260081093,
"claude-2.0": 0.6270890431027705,
"claude-3-5-sonnet-20240620": 0.6142432139552865,
"claude-3-opus-20240229": 0.6198227621200374,
"gemini-1.5-pro-api-0409-preview": 0.5,
"gemini-1.5-pro-exp-0801": 0.5458257442135583,
"gemini-advanced-0514": 0.6025435895577902,
"gemini-pro-dev-api": 0.6233440078256695,
"glm-4-0520": 0.6327707226950309,
"gpt-3.5-turbo-0314": 0.579682755828864,
"gpt-4o-2024-05-13": 0.6036686971453942,
"gpt-4o-2024-08-06": 0.6052143488362124,
"gpt-4o-mini-2024-07-18": 0.6140012780307695,
"llama-2-70b-chat": 0.6389839091289585,
"llama-3.1-70b-instruct": 0.6028024856995498,
"nemotron-4-340b-instruct": 0.6345306684098833,
"yi-large-preview": 0.6386902518991052
},
"gemini-1.5-pro-exp-0801": {
"athene-70b-0725": 0.5772709255434492,
"bard-jan-24-gemini-pro": 0.5483123445198447,
"chatgpt-4o-latest": 0.5017186888746599,
"claude-1": 0.5504227244540455,
"claude-2.0": 0.5832015454454681,
"claude-3-5-sonnet-20240620": 0.569880853157496,
"claude-3-opus-20240229": 0.5756587773809309,
"gemini-1.5-pro-api-0409-preview": 0.4541742557864416,
"gemini-1.5-pro-exp-0801": 0.5,
"gemini-advanced-0514": 0.557804367756876,
"gemini-pro-dev-api": 0.5793114416789945,
"glm-4-0520": 0.5891137641041534,
"gpt-3.5-turbo-0314": 0.5343588600926158,
"gpt-4o-2024-05-13": 0.5589634219132797,
"gpt-4o-2024-08-06": 0.5605565036097535,
"gpt-4o-mini-2024-07-18": 0.5696305884618997,
"llama-2-70b-chat": 0.5955935211645167,
"llama-3.1-70b-instruct": 0.5580710321113055,
"nemotron-4-340b-instruct": 0.5909476824028691,
"yi-large-preview": 0.5952869238841539
},
"gemini-advanced-0514": {
"athene-70b-0725": 0.519820681777397,
"bard-jan-24-gemini-pro": 0.49040074678712725,
"chatgpt-4o-latest": 0.4438920243191068,
"claude-1": 0.4925312817882607,
"claude-2.0": 0.525895343262826,
"claude-3-5-sonnet-20240620": 0.5122748179085177,
"claude-3-opus-20240229": 0.5181723093004097,
"gemini-1.5-pro-api-0409-preview": 0.3974564104422098,
"gemini-1.5-pro-exp-0801": 0.44219563224312397,
"gemini-advanced-0514": 0.5,
"gemini-pro-dev-api": 0.5219088424590324,
"glm-4-0520": 0.5319680879028288,
"gpt-3.5-turbo-0314": 0.4763667410118573,
"gpt-4o-2024-05-13": 0.5011750743841354,
"gpt-4o-2024-08-06": 0.5027912177061772,
"gpt-4o-mini-2024-07-18": 0.5120197361562043,
"llama-2-70b-chat": 0.5386432816976552,
"llama-3.1-70b-instruct": 0.5002702935966019,
"nemotron-4-340b-instruct": 0.5338552462294891,
"yi-large-preview": 0.5383269761251527
},
"gemini-pro-dev-api": {
"athene-70b-0725": 0.4979082058870976,
"bard-jan-24-gemini-pro": 0.4685183877340279,
"chatgpt-4o-latest": 0.4223649169173924,
"claude-1": 0.47064165506714317,
"claude-2.0": 0.5039955681382927,
"claude-3-5-sonnet-20240620": 0.4903556008902106,
"claude-3-opus-20240229": 0.4962575067822826,
"gemini-1.5-pro-api-0409-preview": 0.37665599217433055,
"gemini-1.5-pro-exp-0801": 0.4206885583210056,
"gemini-advanced-0514": 0.4780911575409677,
"gemini-pro-dev-api": 0.5,
"glm-4-0520": 0.5100875059468505,
"gpt-3.5-turbo-0314": 0.45455202627804153,
"gpt-4o-2024-05-13": 0.47926409658161856,
"gpt-4o-2024-08-06": 0.4808776977523424,
"gpt-4o-mini-2024-07-18": 0.49010046598318574,
"llama-2-70b-chat": 0.5167913032942285,
"llama-3.1-70b-instruct": 0.4783609385670775,
"nemotron-4-340b-instruct": 0.5119819532314825,
"yi-large-preview": 0.516473464718453
},
"glm-4-0520": {
"athene-70b-0725": 0.4878217278344273,
"bard-jan-24-gemini-pro": 0.4584836193730801,
"chatgpt-4o-latest": 0.41255135063749104,
"claude-1": 0.4606008217775131,
"claude-2.0": 0.49390707988312654,
"claude-3-5-sonnet-20240620": 0.48027577066139915,
"claude-3-opus-20240229": 0.48617208898399294,
"gemini-1.5-pro-api-0409-preview": 0.3672292773049691,
"gemini-1.5-pro-exp-0801": 0.4108862358958465,
"gemini-advanced-0514": 0.4680319120971712,
"gemini-pro-dev-api": 0.4899124940531495,
"glm-4-0520": 0.5,
"gpt-3.5-turbo-0314": 0.444566176363753,
"gpt-4o-2024-05-13": 0.4692023588423809,
"gpt-4o-2024-08-06": 0.47081271228911237,
"gpt-4o-mini-2024-07-18": 0.4800209406003484,
"llama-2-70b-chat": 0.5067083424471972,
"llama-3.1-70b-instruct": 0.4683011100825741,
"nemotron-4-340b-instruct": 0.5018953636400675,
"yi-large-preview": 0.5063902063717687
},
"gpt-3.5-turbo-0314": {
"athene-70b-0725": 0.5433726729881329,
"bard-jan-24-gemini-pro": 0.5140467524523086,
"chatgpt-4o-latest": 0.4673521170660229,
"claude-1": 0.5161759616621979,
"claude-2.0": 0.5494076540226519,
"claude-3-5-sonnet-20240620": 0.5358664583262915,
"claude-3-opus-20240229": 0.5417338743515567,
"gemini-1.5-pro-api-0409-preview": 0.4203172441711361,
"gemini-1.5-pro-exp-0801": 0.4656411399073841,
"gemini-advanced-0514": 0.5236332589881427,
"gemini-pro-dev-api": 0.5454479737219585,
"glm-4-0520": 0.5554338236362469,
"gpt-3.5-turbo-0314": 0.5,
"gpt-4o-2024-05-13": 0.5248055778856127,
"gpt-4o-2024-08-06": 0.5264175061108259,
"gpt-4o-mini-2024-07-18": 0.5356125299744643,
"llama-2-70b-chat": 0.5620498683755348,
"llama-3.1-70b-instruct": 0.5239029418245605,
"nemotron-4-340b-instruct": 0.5573051037168208,
"yi-large-preview": 0.5617365533684751
},
"gpt-4o-2024-05-13": {
"athene-70b-0725": 0.5186473446377271,
"bard-jan-24-gemini-pro": 0.48922615851167606,
"chatgpt-4o-latest": 0.4427320528773593,
"claude-1": 0.49135651083552223,
"claude-2.0": 0.5247232780927109,
"claude-3-5-sonnet-20240620": 0.511100383964325,
"claude-3-opus-20240229": 0.5169986868635432,
"gemini-1.5-pro-api-0409-preview": 0.3963313028546058,
"gemini-1.5-pro-exp-0801": 0.44103657808672025,
"gemini-advanced-0514": 0.4988249256158645,
"gemini-pro-dev-api": 0.5207359034183815,
"glm-4-0520": 0.5307976411576192,
"gpt-3.5-turbo-0314": 0.4751944221143874,
"gpt-4o-2024-05-13": 0.5,
"gpt-4o-2024-08-06": 0.501616164525399,
"gpt-4o-mini-2024-07-18": 0.5108452744903431,
"llama-2-70b-chat": 0.5374750140847713,
"llama-3.1-70b-instruct": 0.4990952180629769,
"nemotron-4-340b-instruct": 0.5326853730599583,
"yi-large-preview": 0.537158595794847
},
"gpt-4o-2024-08-06": {
"athene-70b-0725": 0.517033233446606,
"bard-jan-24-gemini-pro": 0.48761085688020456,
"chatgpt-4o-latest": 0.44113768022450583,
"claude-1": 0.4897409195588359,
"claude-2.0": 0.5231108073107922,
"claude-3-5-sonnet-20240620": 0.5094849000771304,
"claude-3-opus-20240229": 0.5153842129200518,
"gemini-1.5-pro-api-0409-preview": 0.39478565116378755,
"gemini-1.5-pro-exp-0801": 0.4394434963902465,
"gemini-advanced-0514": 0.4972087822938228,
"gemini-pro-dev-api": 0.5191223022476575,
"glm-4-0520": 0.5291872877108876,
"gpt-3.5-turbo-0314": 0.473582493889174,
"gpt-4o-2024-05-13": 0.4983838354746009,
"gpt-4o-2024-08-06": 0.5,
"gpt-4o-mini-2024-07-18": 0.5092297570723646,
"llama-2-70b-chat": 0.5358675389424615,
"llama-3.1-70b-instruct": 0.49747906828277433,
"nemotron-4-340b-instruct": 0.531075774838379,
"yi-large-preview": 0.5355509712390535
},
"gpt-4o-mini-2024-07-18": {
"athene-70b-0725": 0.507808386681544,
"bard-jan-24-gemini-pro": 0.47839098366657384,
"chatgpt-4o-latest": 0.43205557587413584,
"claude-1": 0.48051854119187704,
"claude-2.0": 0.5138929040607993,
"claude-3-5-sonnet-20240620": 0.5002552323803292,
"claude-3-opus-20240229": 0.5061579533816516,
"gemini-1.5-pro-api-0409-preview": 0.38599872196923046,
"gemini-1.5-pro-exp-0801": 0.4303694115381003,
"gemini-advanced-0514": 0.4879802638437957,
"gemini-pro-dev-api": 0.5098995340168142,
"glm-4-0520": 0.5199790593996516,
"gpt-3.5-turbo-0314": 0.4643874700255357,
"gpt-4o-2024-05-13": 0.4891547255096569,
"gpt-4o-2024-08-06": 0.4907702429276354,
"gpt-4o-mini-2024-07-18": 0.5,
"llama-2-70b-chat": 0.5266731022503263,
"llama-3.1-70b-instruct": 0.4882504047493449,
"nemotron-4-340b-instruct": 0.5218711102154154,
"yi-large-preview": 0.5263558063552674
},
"llama-2-70b-chat": {
"athene-70b-0725": 0.4811195552160165,
"bard-jan-24-gemini-pro": 0.4518289406610546,
"chatgpt-4o-latest": 0.40606343428200253,
"claude-1": 0.4539411733076995,
"claude-2.0": 0.4872008300180327,
"claude-3-5-sonnet-20240620": 0.4735814107159531,
"claude-3-opus-20240229": 0.4794713636760069,
"gemini-1.5-pro-api-0409-preview": 0.3610160908710415,
"gemini-1.5-pro-exp-0801": 0.40440647883548325,
"gemini-advanced-0514": 0.4613567183023448,
"gemini-pro-dev-api": 0.4832086967057715,
"glm-4-0520": 0.4932916575528028,
"gpt-3.5-turbo-0314": 0.43795013162446517,
"gpt-4o-2024-05-13": 0.4625249859152287,
"gpt-4o-2024-08-06": 0.46413246105753847,
"gpt-4o-mini-2024-07-18": 0.47332689774967374,
"llama-2-70b-chat": 0.5,
"llama-3.1-70b-instruct": 0.46162540860366846,
"nemotron-4-340b-instruct": 0.49518677639716263,
"yi-large-preview": 0.49968180936417794
},
"llama-3.1-70b-instruct": {
"athene-70b-0725": 0.5195508071470352,
"bard-jan-24-gemini-pro": 0.4901305556202252,
"chatgpt-4o-latest": 0.4436251505621573,
"claude-1": 0.4922610506835726,
"claude-2.0": 0.5256257671206118,
"claude-3-5-sonnet-20240620": 0.512004683628698,
"claude-3-opus-20240229": 0.5179023674394149,
"gemini-1.5-pro-api-0409-preview": 0.3971975143004503,
"gemini-1.5-pro-exp-0801": 0.4419289678886945,
"gemini-advanced-0514": 0.49972970640339814,
"gemini-pro-dev-api": 0.5216390614329225,
"glm-4-0520": 0.531698889917426,
"gpt-3.5-turbo-0314": 0.47609705817543935,
"gpt-4o-2024-05-13": 0.5009047819370231,
"gpt-4o-2024-08-06": 0.5025209317172257,
"gpt-4o-mini-2024-07-18": 0.5117495952506551,
"llama-2-70b-chat": 0.5383745913963316,
"llama-3.1-70b-instruct": 0.5,
"nemotron-4-340b-instruct": 0.5335861820021834,
"yi-large-preview": 0.5380582595922262
},
"nemotron-4-340b-instruct": {
"athene-70b-0725": 0.4859276634793583,
"bard-jan-24-gemini-pro": 0.45660191547771745,
"chatgpt-4o-latest": 0.4107151817570633,
"claude-1": 0.458717789260918,
"claude-2.0": 0.4920120852303797,
"claude-3-5-sonnet-20240620": 0.47838363949618123,
"claude-3-opus-20240229": 0.4842783735313118,
"gemini-1.5-pro-api-0409-preview": 0.3654693315901167,
"gemini-1.5-pro-exp-0801": 0.4090523175971309,
"gemini-advanced-0514": 0.4661447537705109,
"gemini-pro-dev-api": 0.4880180467685175,
"glm-4-0520": 0.49810463635993246,
"gpt-3.5-turbo-0314": 0.4426948962831791,
"gpt-4o-2024-05-13": 0.4673146269400416,
"gpt-4o-2024-08-06": 0.46892422516162097,
"gpt-4o-mini-2024-07-18": 0.47812888978458457,
"llama-2-70b-chat": 0.5048132236028374,
"llama-3.1-70b-instruct": 0.4664138179978165,
"nemotron-4-340b-instruct": 0.5,
"yi-large-preview": 0.5044950605041638
},
"yi-large-preview": {
"athene-70b-0725": 0.48143729978415806,
"bard-jan-24-gemini-pro": 0.4521441972422958,
"chatgpt-4o-latest": 0.4063704306655477,
"claude-1": 0.45425668238107064,
"claude-2.0": 0.48751881733159297,
"claude-3-5-sonnet-20240620": 0.4738987237070381,
"claude-3-opus-20240229": 0.47978902623790215,
"gemini-1.5-pro-api-0409-preview": 0.3613097481008948,
"gemini-1.5-pro-exp-0801": 0.40471307611584606,
"gemini-advanced-0514": 0.4616730238748474,
"gemini-pro-dev-api": 0.48352653528154693,
"glm-4-0520": 0.4936097936282313,
"gpt-3.5-turbo-0314": 0.4382634466315249,
"gpt-4o-2024-05-13": 0.462841404205153,
"gpt-4o-2024-08-06": 0.46444902876094657,
"gpt-4o-mini-2024-07-18": 0.4736441936447327,
"llama-2-70b-chat": 0.5003181906358221,
"llama-3.1-70b-instruct": 0.4619417404077737,
"nemotron-4-340b-instruct": 0.4955049394958363,
"yi-large-preview": 0.5
}
}
}
chapter7/elo-leaderboard/validation/runs/exp7-7-arena-20260731-v1/win_rate_matrix.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 226 KiB