ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"experiment": "7-7",
|
||||
"manifest_sha256": "b67f8b15a088e694ae356322e0ccd676b54ed79a0dc1c787ca1960ef670ba2e8",
|
||||
"official_complete": true,
|
||||
"run": "runs/exp7-7-arena-20260731-v1",
|
||||
"status": "passed"
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
"""Run the complete, evidence-producing Experiment 7-7 campaign.
|
||||
|
||||
The public Arena file is deliberately not copied into git. A canonical run
|
||||
binds the exact input by URL, size, record count, and SHA-256, then retains all
|
||||
derived tables, visualizations, the D3 history animation, and a manifest that
|
||||
hashes every output and the source used to create it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
os.environ.setdefault("MPLBACKEND", "Agg")
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import seaborn as sns
|
||||
from scipy.stats import kendalltau, spearmanr
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
PROJECT = HERE.parent
|
||||
sys.path.insert(0, str(PROJECT))
|
||||
|
||||
from animation import create_simple_animation
|
||||
from bradley_terry import compute_bradley_terry_leaderboard
|
||||
from optimized_elo import (
|
||||
NumpyEloRatingSystem,
|
||||
process_elo_updates_vectorized,
|
||||
)
|
||||
|
||||
DATASET_URL = (
|
||||
"https://storage.googleapis.com/arena_external_data/public/"
|
||||
"clean_battle_20240814_public.json"
|
||||
)
|
||||
REQUIRED_COLUMNS = ["model_a", "model_b", "winner", "tstamp", "anony", "turn"]
|
||||
ALLOWED_OUTCOMES = {"model_a", "model_b", "tie", "tie (bothbad)"}
|
||||
|
||||
|
||||
def sha256_file(path: Path, chunk_size: int = 8 * 1024 * 1024) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(chunk_size):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, value: Any) -> None:
|
||||
path.write_text(
|
||||
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def json_records(frame: pd.DataFrame) -> list[dict[str, Any]]:
|
||||
return json.loads(frame.to_json(orient="records", date_format="iso"))
|
||||
|
||||
|
||||
def load_and_filter(path: Path, max_records: int) -> tuple[pd.DataFrame, dict[str, Any]]:
|
||||
started = time.perf_counter()
|
||||
raw = pd.read_json(path)
|
||||
missing = sorted(set(REQUIRED_COLUMNS) - set(raw.columns))
|
||||
if missing:
|
||||
raise ValueError(f"Arena input is missing columns: {missing}")
|
||||
|
||||
source_records = len(raw)
|
||||
frame = raw[REQUIRED_COLUMNS + (["dedup_tag"] if "dedup_tag" in raw else [])].copy()
|
||||
del raw
|
||||
frame = frame[frame["anony"].eq(True) & frame["turn"].ge(1)]
|
||||
if "dedup_tag" in frame:
|
||||
sampled = frame["dedup_tag"].map(
|
||||
lambda value: bool(value.get("sampled", False)) if isinstance(value, dict) else False
|
||||
)
|
||||
frame = frame[sampled]
|
||||
frame = frame[frame["winner"].isin(ALLOWED_OUTCOMES)]
|
||||
frame = frame.sort_values("tstamp", kind="stable").reset_index(drop=True)
|
||||
if max_records:
|
||||
frame = frame.head(max_records).copy()
|
||||
if frame.empty:
|
||||
raise ValueError("Arena filtering produced no accepted blind votes")
|
||||
|
||||
metadata = {
|
||||
"source_records": source_records,
|
||||
"accepted_records": len(frame),
|
||||
"model_count": len(set(frame["model_a"]) | set(frame["model_b"])),
|
||||
"start_utc": datetime.fromtimestamp(float(frame["tstamp"].min()), timezone.utc).isoformat(),
|
||||
"end_utc": datetime.fromtimestamp(float(frame["tstamp"].max()), timezone.utc).isoformat(),
|
||||
"outcomes": {str(k): int(v) for k, v in frame["winner"].value_counts().items()},
|
||||
"load_filter_seconds": round(time.perf_counter() - started, 3),
|
||||
"bounded_test_run": bool(max_records),
|
||||
}
|
||||
return frame, metadata
|
||||
|
||||
|
||||
def online_elo_and_history(
|
||||
frame: pd.DataFrame,
|
||||
) -> tuple[pd.DataFrame, pd.DataFrame, NumpyEloRatingSystem, float]:
|
||||
started = time.perf_counter()
|
||||
system = NumpyEloRatingSystem(initial_rating=1000.0, k_factor=4.0)
|
||||
model_a, model_b, outcomes = system._prepare_data(frame)
|
||||
|
||||
months = (
|
||||
pd.to_datetime(frame["tstamp"], unit="s", utc=True)
|
||||
.dt.tz_localize(None)
|
||||
.dt.to_period("M")
|
||||
)
|
||||
boundaries = np.flatnonzero(months.to_numpy()[1:] != months.to_numpy()[:-1]) + 1
|
||||
boundaries = np.append(boundaries, len(frame))
|
||||
start = 0
|
||||
history_rows: list[dict[str, Any]] = []
|
||||
for stop in boundaries:
|
||||
process_elo_updates_vectorized(
|
||||
system.ratings,
|
||||
model_a[start:stop],
|
||||
model_b[start:stop],
|
||||
outcomes[start:stop],
|
||||
system.k_factor,
|
||||
system.match_counts,
|
||||
system.win_counts,
|
||||
)
|
||||
snapshot = system.get_leaderboard()
|
||||
date = pd.to_datetime(float(frame.iloc[stop - 1]["tstamp"]), unit="s", utc=True)
|
||||
for rank, (model, rating, matches, wins) in enumerate(snapshot, 1):
|
||||
history_rows.append(
|
||||
{
|
||||
"date": date.tz_localize(None),
|
||||
"model": model,
|
||||
"rating": rating,
|
||||
"rank": rank,
|
||||
"matches": matches,
|
||||
"wins": wins,
|
||||
}
|
||||
)
|
||||
start = int(stop)
|
||||
|
||||
leaderboard = pd.DataFrame(
|
||||
system.get_leaderboard(), columns=["model", "rating", "matches", "wins"]
|
||||
)
|
||||
leaderboard.insert(0, "rank", range(1, len(leaderboard) + 1))
|
||||
history = pd.DataFrame(history_rows)
|
||||
return leaderboard, history, system, round(time.perf_counter() - started, 3)
|
||||
|
||||
|
||||
def rank_comparison(online: pd.DataFrame, official_method: pd.DataFrame) -> dict[str, Any]:
|
||||
online_rank = online.set_index("model")["rank"]
|
||||
official = official_method.sort_values("rating", ascending=False).reset_index(drop=True)
|
||||
official["rank"] = np.arange(1, len(official) + 1)
|
||||
official_rank = official.set_index("model")["rank"]
|
||||
common = sorted(set(online_rank.index) & set(official_rank.index))
|
||||
rho = spearmanr(online_rank.loc[common], official_rank.loc[common]).statistic
|
||||
tau = kendalltau(online_rank.loc[common], official_rank.loc[common]).statistic
|
||||
online_top = online.nsmallest(20, "rank")["model"].tolist()
|
||||
official_top = official.nsmallest(20, "rank")["model"].tolist()
|
||||
return {
|
||||
"comparison_target": "Bradley-Terry MLE reconstruction used by Chatbot Arena",
|
||||
"claim_boundary": (
|
||||
"This is a same-snapshot reconstruction of the official method, not a scrape of "
|
||||
"the mutable live leaderboard. Scores need not match the live service."
|
||||
),
|
||||
"common_models": len(common),
|
||||
"spearman_rank_correlation": round(float(rho), 6),
|
||||
"kendall_rank_correlation": round(float(tau), 6),
|
||||
"top_20_overlap": len(set(online_top) & set(official_top)),
|
||||
"online_top_20": online_top,
|
||||
"official_method_top_20": official_top,
|
||||
}
|
||||
|
||||
|
||||
def empirical_matrix(frame: pd.DataFrame, models: list[str]) -> pd.DataFrame:
|
||||
subset = frame[frame["model_a"].isin(models) & frame["model_b"].isin(models)].copy()
|
||||
rows: list[tuple[str, str, float]] = []
|
||||
for a, b, winner in subset[["model_a", "model_b", "winner"]].itertuples(index=False):
|
||||
score = 1.0 if winner == "model_a" else 0.0 if winner == "model_b" else 0.5
|
||||
rows.append((a, b, score))
|
||||
rows.append((b, a, 1.0 - score))
|
||||
scored = pd.DataFrame(rows, columns=["model", "opponent", "score"])
|
||||
matrix = scored.pivot_table(index="model", columns="opponent", values="score", aggfunc="mean")
|
||||
matrix = matrix.reindex(index=models, columns=models)
|
||||
np.fill_diagonal(matrix.values, 0.5)
|
||||
return matrix
|
||||
|
||||
|
||||
def plot_artifacts(
|
||||
out: Path,
|
||||
online: pd.DataFrame,
|
||||
history: pd.DataFrame,
|
||||
empirical: pd.DataFrame,
|
||||
) -> None:
|
||||
top = online.head(20).sort_values("rating")
|
||||
fig, ax = plt.subplots(figsize=(11, 8))
|
||||
ax.barh(top["model"], top["rating"], color="#3b82f6")
|
||||
ax.set_title("Experiment 7-7: Online Elo leaderboard")
|
||||
ax.set_xlabel("Elo rating (K=4, chronological)")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out / "leaderboard.png", dpi=180)
|
||||
plt.close(fig)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(13, 11))
|
||||
sns.heatmap(empirical, cmap="RdYlGn", center=0.5, vmin=0, vmax=1, ax=ax)
|
||||
ax.set_title("Empirical pairwise win rate — final online-Elo top 20")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out / "win_rate_matrix.png", dpi=180)
|
||||
plt.close(fig)
|
||||
|
||||
top_models = online.head(10)["model"].tolist()
|
||||
fig, ax = plt.subplots(figsize=(13, 7))
|
||||
for model in top_models:
|
||||
values = history[history["model"].eq(model)].sort_values("date")
|
||||
ax.plot(values["date"], values["rating"], label=model, linewidth=1.8)
|
||||
ax.set_title("Monthly online-Elo evolution — final top 10")
|
||||
ax.set_ylabel("Elo rating")
|
||||
ax.legend(fontsize=7, ncol=2)
|
||||
fig.autofmt_xdate()
|
||||
fig.tight_layout()
|
||||
fig.savefig(out / "rating_history.png", dpi=180)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--input", type=Path, required=True, help="Downloaded public Arena JSON")
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--bootstrap-rounds", type=int, default=20)
|
||||
parser.add_argument("--max-records", type=int, default=0, help="Noncanonical bounded test only")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
input_path = args.input.resolve()
|
||||
if not input_path.is_file():
|
||||
raise SystemExit(f"Arena input not found: {input_path}")
|
||||
|
||||
run_started = time.perf_counter()
|
||||
input_hash = sha256_file(input_path)
|
||||
frame, dataset = load_and_filter(input_path, args.max_records)
|
||||
online, history, online_system, online_seconds = online_elo_and_history(frame)
|
||||
|
||||
bt_started = time.perf_counter()
|
||||
official_method = compute_bradley_terry_leaderboard(
|
||||
frame[["model_a", "model_b", "winner"]],
|
||||
bootstrap_rounds=args.bootstrap_rounds,
|
||||
)
|
||||
bt_seconds = round(time.perf_counter() - bt_started, 3)
|
||||
official_method = official_method.sort_values("rating", ascending=False).reset_index(drop=True)
|
||||
official_method.insert(0, "rank", range(1, len(official_method) + 1))
|
||||
|
||||
comparison = rank_comparison(online, official_method)
|
||||
top_models = online.head(20)["model"].tolist()
|
||||
empirical = empirical_matrix(frame, top_models)
|
||||
predicted = pd.DataFrame(
|
||||
{
|
||||
opponent: {
|
||||
model: online_system.calculate_win_probability(model, opponent)
|
||||
for model in top_models
|
||||
}
|
||||
for opponent in top_models
|
||||
}
|
||||
).reindex(index=top_models, columns=top_models)
|
||||
|
||||
write_json(args.output_dir / "online_elo.json", json_records(online))
|
||||
write_json(args.output_dir / "bradley_terry.json", json_records(official_method))
|
||||
write_json(
|
||||
args.output_dir / "win_rate_matrix.json",
|
||||
{
|
||||
"models": top_models,
|
||||
"empirical": empirical.where(pd.notna(empirical), None).to_dict(orient="index"),
|
||||
"online_elo_predicted": predicted.to_dict(orient="index"),
|
||||
},
|
||||
)
|
||||
write_json(args.output_dir / "rating_history.json", json_records(history))
|
||||
plot_artifacts(args.output_dir, online, history, empirical)
|
||||
create_simple_animation(history, str(args.output_dir / "leaderboard_animation.html"), top_n=15)
|
||||
|
||||
gates = {
|
||||
"official_public_arena_snapshot_hashed": not args.max_records,
|
||||
"millions_of_blind_votes_loaded": dataset["source_records"] >= 1_000_000,
|
||||
"chronological_online_elo_k4_completed": len(online) == dataset["model_count"],
|
||||
"bradley_terry_official_method_completed": len(official_method) == dataset["model_count"],
|
||||
"online_vs_official_method_rank_agreement_observed": (
|
||||
comparison["spearman_rank_correlation"] >= 0.70
|
||||
and comparison["top_20_overlap"] >= 10
|
||||
),
|
||||
"pairwise_empirical_and_predicted_matrix_saved": len(empirical) == 20,
|
||||
"monthly_history_saved": history["date"].nunique() >= 2,
|
||||
"d3_animation_saved": (args.output_dir / "leaderboard_animation.html").is_file(),
|
||||
"static_visualizations_saved": all(
|
||||
(args.output_dir / name).is_file()
|
||||
for name in ["leaderboard.png", "win_rate_matrix.png", "rating_history.png"]
|
||||
),
|
||||
}
|
||||
accepted = all(gates.values())
|
||||
summary = {
|
||||
"schema_version": 1,
|
||||
"experiment": "7-7",
|
||||
"status": "passed" if accepted else "noncanonical_test",
|
||||
"official_complete": accepted,
|
||||
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"dataset": {
|
||||
"url": DATASET_URL,
|
||||
"path_recorded_as": input_path.name,
|
||||
"bytes": input_path.stat().st_size,
|
||||
"sha256": input_hash,
|
||||
**dataset,
|
||||
},
|
||||
"protocol": {
|
||||
"online_elo": "initial=1000, K=4, stable chronological order",
|
||||
"official_method": "Bradley-Terry maximum-likelihood reconstruction",
|
||||
"history_interval": "monthly cumulative snapshots",
|
||||
"bootstrap_rounds": args.bootstrap_rounds,
|
||||
"bootstrap_random_seed": 0,
|
||||
},
|
||||
"results": {
|
||||
"online_top_20": json_records(online.head(20)),
|
||||
"official_method_top_20": json_records(official_method.head(20)),
|
||||
"rank_comparison": comparison,
|
||||
},
|
||||
"timing_seconds": {
|
||||
"online_and_history": online_seconds,
|
||||
"bradley_terry": bt_seconds,
|
||||
"total": round(time.perf_counter() - run_started, 3),
|
||||
},
|
||||
"gates": gates,
|
||||
}
|
||||
write_json(args.output_dir / "summary.json", summary)
|
||||
|
||||
artifact_names = [
|
||||
"online_elo.json",
|
||||
"bradley_terry.json",
|
||||
"win_rate_matrix.json",
|
||||
"rating_history.json",
|
||||
"leaderboard.png",
|
||||
"win_rate_matrix.png",
|
||||
"rating_history.png",
|
||||
"leaderboard_animation.html",
|
||||
"summary.json",
|
||||
]
|
||||
source_names = [
|
||||
"animation.py",
|
||||
"bradley_terry.py",
|
||||
"optimized_elo.py",
|
||||
"validation/run_experiment.py",
|
||||
"validation/validate_evidence.py",
|
||||
]
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"experiment": "7-7",
|
||||
"status": summary["status"],
|
||||
"official_complete": accepted,
|
||||
"input": {
|
||||
"url": DATASET_URL,
|
||||
"filename": input_path.name,
|
||||
"bytes": input_path.stat().st_size,
|
||||
"sha256": input_hash,
|
||||
},
|
||||
"artifacts": {
|
||||
name: {"bytes": (args.output_dir / name).stat().st_size, "sha256": sha256_file(args.output_dir / name)}
|
||||
for name in artifact_names
|
||||
},
|
||||
"sources": {
|
||||
name: sha256_file(PROJECT / name)
|
||||
for name in source_names
|
||||
},
|
||||
"runtime": {
|
||||
"python": sys.version.split()[0],
|
||||
"platform": platform.platform(),
|
||||
"pandas": pd.__version__,
|
||||
"numpy": np.__version__,
|
||||
},
|
||||
"gates": gates,
|
||||
}
|
||||
write_json(args.output_dir / "manifest.json", manifest)
|
||||
print(json.dumps({"status": manifest["status"], "output": str(args.output_dir)}, indent=2))
|
||||
if not accepted:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,905 @@
|
||||
[
|
||||
{
|
||||
"lower_ci": 1199.4356201692,
|
||||
"model": "chatgpt-4o-latest",
|
||||
"rank": 1,
|
||||
"rating": 1202.8722324768,
|
||||
"upper_ci": 1207.3805726125
|
||||
},
|
||||
{
|
||||
"lower_ci": 1182.767114327,
|
||||
"model": "gemini-1.5-pro-exp-0801",
|
||||
"rank": 2,
|
||||
"rating": 1187.3727762743,
|
||||
"upper_ci": 1191.7564879767
|
||||
},
|
||||
{
|
||||
"lower_ci": 1173.1270556478,
|
||||
"model": "gpt-4o-2024-05-13",
|
||||
"rank": 3,
|
||||
"rating": 1174.5475812185,
|
||||
"upper_ci": 1177.0403252386
|
||||
},
|
||||
{
|
||||
"lower_ci": 1161.1978031325,
|
||||
"model": "gpt-4o-mini-2024-07-18",
|
||||
"rank": 4,
|
||||
"rating": 1162.9712796813,
|
||||
"upper_ci": 1166.568334134
|
||||
},
|
||||
{
|
||||
"lower_ci": 1156.0224735749,
|
||||
"model": "claude-3-5-sonnet-20240620",
|
||||
"rank": 5,
|
||||
"rating": 1159.6692177384,
|
||||
"upper_ci": 1161.8458744744
|
||||
},
|
||||
{
|
||||
"lower_ci": 1153.6447817793,
|
||||
"model": "gemini-advanced-0514",
|
||||
"rank": 6,
|
||||
"rating": 1155.6305094344,
|
||||
"upper_ci": 1157.1030226933
|
||||
},
|
||||
{
|
||||
"lower_ci": 1150.4614200682,
|
||||
"model": "llama-3.1-405b-instruct",
|
||||
"rank": 7,
|
||||
"rating": 1153.3950109278,
|
||||
"upper_ci": 1157.0848375454
|
||||
},
|
||||
{
|
||||
"lower_ci": 1146.348878889,
|
||||
"model": "gpt-4o-2024-08-06",
|
||||
"rank": 8,
|
||||
"rating": 1150.7428051987,
|
||||
"upper_ci": 1156.3245456986
|
||||
},
|
||||
{
|
||||
"lower_ci": 1146.7772301944,
|
||||
"model": "gemini-1.5-pro-api-0514",
|
||||
"rank": 9,
|
||||
"rating": 1149.1265569601,
|
||||
"upper_ci": 1151.5010539806
|
||||
},
|
||||
{
|
||||
"lower_ci": 1143.6682527667,
|
||||
"model": "gemini-1.5-pro-api-0409-preview",
|
||||
"rank": 10,
|
||||
"rating": 1146.1756037638,
|
||||
"upper_ci": 1147.8895040945
|
||||
},
|
||||
{
|
||||
"lower_ci": 1142.3990049162,
|
||||
"model": "gpt-4-turbo-2024-04-09",
|
||||
"rank": 11,
|
||||
"rating": 1144.9187340632,
|
||||
"upper_ci": 1146.8362809167
|
||||
},
|
||||
{
|
||||
"lower_ci": 1138.1303324947,
|
||||
"model": "gpt-4-1106-preview",
|
||||
"rank": 12,
|
||||
"rating": 1139.5004505155,
|
||||
"upper_ci": 1140.97644635
|
||||
},
|
||||
{
|
||||
"lower_ci": 1136.3852091186,
|
||||
"model": "mistral-large-2407",
|
||||
"rank": 13,
|
||||
"rating": 1139.3566231011,
|
||||
"upper_ci": 1143.8960727747
|
||||
},
|
||||
{
|
||||
"lower_ci": 1135.0624903381,
|
||||
"model": "athene-70b-0725",
|
||||
"rank": 14,
|
||||
"rating": 1138.2016223593,
|
||||
"upper_ci": 1141.397916447
|
||||
},
|
||||
{
|
||||
"lower_ci": 1135.0278139027,
|
||||
"model": "claude-3-opus-20240229",
|
||||
"rank": 15,
|
||||
"rating": 1136.5739464324,
|
||||
"upper_ci": 1137.5663039742
|
||||
},
|
||||
{
|
||||
"lower_ci": 1131.7219814044,
|
||||
"model": "llama-3.1-70b-instruct",
|
||||
"rank": 16,
|
||||
"rating": 1134.8942150397,
|
||||
"upper_ci": 1139.3028524197
|
||||
},
|
||||
{
|
||||
"lower_ci": 1132.1800230236,
|
||||
"model": "gpt-4-0125-preview",
|
||||
"rank": 17,
|
||||
"rating": 1133.974390087,
|
||||
"upper_ci": 1135.7552442263
|
||||
},
|
||||
{
|
||||
"lower_ci": 1127.3300289872,
|
||||
"model": "yi-large-preview",
|
||||
"rank": 18,
|
||||
"rating": 1128.1692229165,
|
||||
"upper_ci": 1131.371743138
|
||||
},
|
||||
{
|
||||
"lower_ci": 1111.850158929,
|
||||
"model": "reka-core-20240722",
|
||||
"rank": 19,
|
||||
"rating": 1117.1464671073,
|
||||
"upper_ci": 1121.9537941313
|
||||
},
|
||||
{
|
||||
"lower_ci": 1114.0244965884,
|
||||
"model": "gemini-1.5-flash-api-0514",
|
||||
"rank": 20,
|
||||
"rating": 1116.6161359294,
|
||||
"upper_ci": 1118.5223962538
|
||||
},
|
||||
{
|
||||
"lower_ci": 1103.6704090106,
|
||||
"model": "deepseek-v2-api-0628",
|
||||
"rank": 21,
|
||||
"rating": 1107.2808888587,
|
||||
"upper_ci": 1112.0417779712
|
||||
},
|
||||
{
|
||||
"lower_ci": 1103.9432951297,
|
||||
"model": "gemma-2-27b-it",
|
||||
"rank": 22,
|
||||
"rating": 1105.9132519337,
|
||||
"upper_ci": 1108.0293102426
|
||||
},
|
||||
{
|
||||
"lower_ci": 1099.3459906048,
|
||||
"model": "deepseek-coder-v2-0724",
|
||||
"rank": 23,
|
||||
"rating": 1105.3954927337,
|
||||
"upper_ci": 1113.0353361228
|
||||
},
|
||||
{
|
||||
"lower_ci": 1099.9821845459,
|
||||
"model": "yi-large",
|
||||
"rank": 24,
|
||||
"rating": 1102.4018397329,
|
||||
"upper_ci": 1104.8525831946
|
||||
},
|
||||
{
|
||||
"lower_ci": 1094.5198632757,
|
||||
"model": "nemotron-4-340b-instruct",
|
||||
"rank": 25,
|
||||
"rating": 1098.0765768899,
|
||||
"upper_ci": 1100.3082488796
|
||||
},
|
||||
{
|
||||
"lower_ci": 1092.3225621341,
|
||||
"model": "bard-jan-24-gemini-pro",
|
||||
"rank": 26,
|
||||
"rating": 1096.589845604,
|
||||
"upper_ci": 1104.5333605578
|
||||
},
|
||||
{
|
||||
"lower_ci": 1089.9500461084,
|
||||
"model": "glm-4-0520",
|
||||
"rank": 27,
|
||||
"rating": 1095.5341569364,
|
||||
"upper_ci": 1100.1005750187
|
||||
},
|
||||
{
|
||||
"lower_ci": 1093.6731280724,
|
||||
"model": "llama-3-70b-instruct",
|
||||
"rank": 28,
|
||||
"rating": 1094.9281802329,
|
||||
"upper_ci": 1095.7283536945
|
||||
},
|
||||
{
|
||||
"lower_ci": 1088.0897564085,
|
||||
"model": "claude-3-sonnet-20240229",
|
||||
"rank": 29,
|
||||
"rating": 1090.1434308802,
|
||||
"upper_ci": 1091.5131310821
|
||||
},
|
||||
{
|
||||
"lower_ci": 1086.224736458,
|
||||
"model": "reka-core-20240501",
|
||||
"rank": 30,
|
||||
"rating": 1088.4321613308,
|
||||
"upper_ci": 1090.272779249
|
||||
},
|
||||
{
|
||||
"lower_ci": 1082.4844014079,
|
||||
"model": "reka-flash-20240722",
|
||||
"rank": 31,
|
||||
"rating": 1088.2114217704,
|
||||
"upper_ci": 1094.1409244278
|
||||
},
|
||||
{
|
||||
"lower_ci": 1076.2505646464,
|
||||
"model": "command-r-plus",
|
||||
"rank": 32,
|
||||
"rating": 1078.6392855764,
|
||||
"upper_ci": 1080.5241685262
|
||||
},
|
||||
{
|
||||
"lower_ci": 1072.978332193,
|
||||
"model": "gemma-2-9b-it",
|
||||
"rank": 33,
|
||||
"rating": 1076.1200490762,
|
||||
"upper_ci": 1078.284506667
|
||||
},
|
||||
{
|
||||
"lower_ci": 1073.5327980749,
|
||||
"model": "qwen2-72b-instruct",
|
||||
"rank": 34,
|
||||
"rating": 1075.9397511888,
|
||||
"upper_ci": 1077.9102798056
|
||||
},
|
||||
{
|
||||
"lower_ci": 1073.0022184434,
|
||||
"model": "gpt-4-0314",
|
||||
"rank": 35,
|
||||
"rating": 1074.4688060969,
|
||||
"upper_ci": 1077.3841437021
|
||||
},
|
||||
{
|
||||
"lower_ci": 1069.6849684012,
|
||||
"model": "qwen-max-0428",
|
||||
"rank": 36,
|
||||
"rating": 1072.288830872,
|
||||
"upper_ci": 1075.2788486953
|
||||
},
|
||||
{
|
||||
"lower_ci": 1067.4064473159,
|
||||
"model": "glm-4-0116",
|
||||
"rank": 37,
|
||||
"rating": 1072.1854778301,
|
||||
"upper_ci": 1078.0278828583
|
||||
},
|
||||
{
|
||||
"lower_ci": 1065.2785446605,
|
||||
"model": "claude-3-haiku-20240307",
|
||||
"rank": 38,
|
||||
"rating": 1067.2876085746,
|
||||
"upper_ci": 1068.5180263057
|
||||
},
|
||||
{
|
||||
"lower_ci": 1063.4499078695,
|
||||
"model": "deepseek-coder-v2",
|
||||
"rank": 39,
|
||||
"rating": 1066.5634197273,
|
||||
"upper_ci": 1071.0372811761
|
||||
},
|
||||
{
|
||||
"lower_ci": 1052.4060997176,
|
||||
"model": "llama-3.1-8b-instruct",
|
||||
"rank": 40,
|
||||
"rating": 1057.2617029177,
|
||||
"upper_ci": 1062.4851344029
|
||||
},
|
||||
{
|
||||
"lower_ci": 1050.6516231153,
|
||||
"model": "reka-flash-preview-20240611",
|
||||
"rank": 41,
|
||||
"rating": 1053.4286082128,
|
||||
"upper_ci": 1056.9047978837
|
||||
},
|
||||
{
|
||||
"lower_ci": 1049.878634752,
|
||||
"model": "gpt-4-0613",
|
||||
"rank": 42,
|
||||
"rating": 1051.2288727429,
|
||||
"upper_ci": 1053.3859970522
|
||||
},
|
||||
{
|
||||
"lower_ci": 1047.658821981,
|
||||
"model": "qwen1.5-110b-chat",
|
||||
"rank": 43,
|
||||
"rating": 1050.1472801797,
|
||||
"upper_ci": 1054.1870464263
|
||||
},
|
||||
{
|
||||
"lower_ci": 1043.4416581432,
|
||||
"model": "yi-1.5-34b-chat",
|
||||
"rank": 44,
|
||||
"rating": 1046.5798307425,
|
||||
"upper_ci": 1048.4870877901
|
||||
},
|
||||
{
|
||||
"lower_ci": 1043.9717227179,
|
||||
"model": "mistral-large-2402",
|
||||
"rank": 45,
|
||||
"rating": 1045.8549746499,
|
||||
"upper_ci": 1048.6558688014
|
||||
},
|
||||
{
|
||||
"lower_ci": 1039.8435107486,
|
||||
"model": "reka-flash-21b-20240226-online",
|
||||
"rank": 46,
|
||||
"rating": 1044.4541273285,
|
||||
"upper_ci": 1047.5722059799
|
||||
},
|
||||
{
|
||||
"lower_ci": 1038.3457593513,
|
||||
"model": "llama-3-8b-instruct",
|
||||
"rank": 47,
|
||||
"rating": 1040.8943517232,
|
||||
"upper_ci": 1042.1727089946
|
||||
},
|
||||
{
|
||||
"lower_ci": 1034.3532315209,
|
||||
"model": "command-r",
|
||||
"rank": 48,
|
||||
"rating": 1037.5954245649,
|
||||
"upper_ci": 1039.801976874
|
||||
},
|
||||
{
|
||||
"lower_ci": 1033.8210405932,
|
||||
"model": "claude-1",
|
||||
"rank": 49,
|
||||
"rating": 1037.2264989242,
|
||||
"upper_ci": 1039.8445566211
|
||||
},
|
||||
{
|
||||
"lower_ci": 1032.4181357042,
|
||||
"model": "reka-flash-21b-20240226",
|
||||
"rank": 50,
|
||||
"rating": 1036.2083492306,
|
||||
"upper_ci": 1038.9143450059
|
||||
},
|
||||
{
|
||||
"lower_ci": 1033.5592570692,
|
||||
"model": "mistral-medium",
|
||||
"rank": 51,
|
||||
"rating": 1035.9830773208,
|
||||
"upper_ci": 1038.0506828269
|
||||
},
|
||||
{
|
||||
"lower_ci": 1033.4716682328,
|
||||
"model": "mixtral-8x22b-instruct-v0.1",
|
||||
"rank": 52,
|
||||
"rating": 1035.6187089038,
|
||||
"upper_ci": 1037.3385699205
|
||||
},
|
||||
{
|
||||
"lower_ci": 1032.3748800585,
|
||||
"model": "qwen1.5-72b-chat",
|
||||
"rank": 53,
|
||||
"rating": 1035.5734020095,
|
||||
"upper_ci": 1037.8584457516
|
||||
},
|
||||
{
|
||||
"lower_ci": 1016.6859168483,
|
||||
"model": "claude-2.0",
|
||||
"rank": 54,
|
||||
"rating": 1020.4032512576,
|
||||
"upper_ci": 1023.0900087939
|
||||
},
|
||||
{
|
||||
"lower_ci": 1016.4818650296,
|
||||
"model": "gemini-pro-dev-api",
|
||||
"rank": 55,
|
||||
"rating": 1018.8216738996,
|
||||
"upper_ci": 1022.0575603351
|
||||
},
|
||||
{
|
||||
"lower_ci": 1013.9644553596,
|
||||
"model": "gemma-2-2b-it",
|
||||
"rank": 56,
|
||||
"rating": 1018.7496539694,
|
||||
"upper_ci": 1022.6257555787
|
||||
},
|
||||
{
|
||||
"lower_ci": 1009.8189712109,
|
||||
"model": "zephyr-orpo-141b-A35b-v0.1",
|
||||
"rank": 57,
|
||||
"rating": 1017.4768858149,
|
||||
"upper_ci": 1023.4231756618
|
||||
},
|
||||
{
|
||||
"lower_ci": 1011.7444717478,
|
||||
"model": "qwen1.5-32b-chat",
|
||||
"rank": 58,
|
||||
"rating": 1014.2192047782,
|
||||
"upper_ci": 1017.4848181067
|
||||
},
|
||||
{
|
||||
"lower_ci": 1009.527805507,
|
||||
"model": "mistral-next",
|
||||
"rank": 59,
|
||||
"rating": 1013.8190347615,
|
||||
"upper_ci": 1016.756252395
|
||||
},
|
||||
{
|
||||
"lower_ci": 1008.0043265533,
|
||||
"model": "phi-3-medium-4k-instruct",
|
||||
"rank": 60,
|
||||
"rating": 1011.6064259691,
|
||||
"upper_ci": 1014.8977711664
|
||||
},
|
||||
{
|
||||
"lower_ci": 1003.3664596337,
|
||||
"model": "claude-2.1",
|
||||
"rank": 61,
|
||||
"rating": 1006.9894951694,
|
||||
"upper_ci": 1008.7578243389
|
||||
},
|
||||
{
|
||||
"lower_ci": 1003.5743630599,
|
||||
"model": "starling-lm-7b-beta",
|
||||
"rank": 62,
|
||||
"rating": 1006.9783218473,
|
||||
"upper_ci": 1009.6647795891
|
||||
},
|
||||
{
|
||||
"lower_ci": 1003.3560004715,
|
||||
"model": "gpt-3.5-turbo-0613",
|
||||
"rank": 63,
|
||||
"rating": 1004.9730864576,
|
||||
"upper_ci": 1008.7646121872
|
||||
},
|
||||
{
|
||||
"lower_ci": 1000.3011942689,
|
||||
"model": "mixtral-8x7b-instruct-v0.1",
|
||||
"rank": 64,
|
||||
"rating": 1002.12545233,
|
||||
"upper_ci": 1004.4555563028
|
||||
},
|
||||
{
|
||||
"lower_ci": 996.8355186436,
|
||||
"model": "yi-34b-chat",
|
||||
"rank": 65,
|
||||
"rating": 1000.1263930743,
|
||||
"upper_ci": 1002.9767969377
|
||||
},
|
||||
{
|
||||
"lower_ci": 996.5513388826,
|
||||
"model": "claude-instant-1",
|
||||
"rank": 66,
|
||||
"rating": 999.4536204497,
|
||||
"upper_ci": 1003.0420312018
|
||||
},
|
||||
{
|
||||
"lower_ci": 991.5815494729,
|
||||
"model": "gemini-pro",
|
||||
"rank": 67,
|
||||
"rating": 998.3909699055,
|
||||
"upper_ci": 1005.6665144072
|
||||
},
|
||||
{
|
||||
"lower_ci": 994.5272838271,
|
||||
"model": "qwen1.5-14b-chat",
|
||||
"rank": 68,
|
||||
"rating": 997.6714948516,
|
||||
"upper_ci": 1000.3276398407
|
||||
},
|
||||
{
|
||||
"lower_ci": 988.7882364736,
|
||||
"model": "gpt-3.5-turbo-0314",
|
||||
"rank": 69,
|
||||
"rating": 997.4790766208,
|
||||
"upper_ci": 1005.2920277339
|
||||
},
|
||||
{
|
||||
"lower_ci": 989.2933547328,
|
||||
"model": "wizardlm-70b",
|
||||
"rank": 70,
|
||||
"rating": 995.5069328559,
|
||||
"upper_ci": 1000.0203225061
|
||||
},
|
||||
{
|
||||
"lower_ci": 992.3677866278,
|
||||
"model": "gpt-3.5-turbo-0125",
|
||||
"rank": 71,
|
||||
"rating": 994.221194786,
|
||||
"upper_ci": 996.0922832004
|
||||
},
|
||||
{
|
||||
"lower_ci": 989.517752069,
|
||||
"model": "dbrx-instruct-preview",
|
||||
"rank": 72,
|
||||
"rating": 991.6558051925,
|
||||
"upper_ci": 994.3414735308
|
||||
},
|
||||
{
|
||||
"lower_ci": 988.3935167102,
|
||||
"model": "phi-3-small-8k-instruct",
|
||||
"rank": 73,
|
||||
"rating": 990.3926872181,
|
||||
"upper_ci": 993.4532484928
|
||||
},
|
||||
{
|
||||
"lower_ci": 980.2003929739,
|
||||
"model": "tulu-2-dpo-70b",
|
||||
"rank": 74,
|
||||
"rating": 986.8619167507,
|
||||
"upper_ci": 995.6082960114
|
||||
},
|
||||
{
|
||||
"lower_ci": 979.2393563337,
|
||||
"model": "llama-2-70b-chat",
|
||||
"rank": 75,
|
||||
"rating": 982.0382072414,
|
||||
"upper_ci": 983.1882888775
|
||||
},
|
||||
{
|
||||
"lower_ci": 976.2082709482,
|
||||
"model": "openchat-3.5-0106",
|
||||
"rank": 76,
|
||||
"rating": 980.6486425506,
|
||||
"upper_ci": 984.7448454329
|
||||
},
|
||||
{
|
||||
"lower_ci": 976.21123294,
|
||||
"model": "vicuna-33b",
|
||||
"rank": 77,
|
||||
"rating": 978.8419077515,
|
||||
"upper_ci": 983.0972163635
|
||||
},
|
||||
{
|
||||
"lower_ci": 976.387496134,
|
||||
"model": "snowflake-arctic-instruct",
|
||||
"rank": 78,
|
||||
"rating": 978.4458039182,
|
||||
"upper_ci": 981.5060581663
|
||||
},
|
||||
{
|
||||
"lower_ci": 969.5806601345,
|
||||
"model": "starling-lm-7b-alpha",
|
||||
"rank": 79,
|
||||
"rating": 976.025808172,
|
||||
"upper_ci": 980.086762811
|
||||
},
|
||||
{
|
||||
"lower_ci": 965.8992419629,
|
||||
"model": "nous-hermes-2-mixtral-8x7b-dpo",
|
||||
"rank": 80,
|
||||
"rating": 975.4699172429,
|
||||
"upper_ci": 981.307231701
|
||||
},
|
||||
{
|
||||
"lower_ci": 969.0894947313,
|
||||
"model": "gemma-1.1-7b-it",
|
||||
"rank": 81,
|
||||
"rating": 971.847848073,
|
||||
"upper_ci": 975.4703266199
|
||||
},
|
||||
{
|
||||
"lower_ci": 966.6172052307,
|
||||
"model": "llama2-70b-steerlm-chat",
|
||||
"rank": 82,
|
||||
"rating": 971.6537111318,
|
||||
"upper_ci": 979.8268878882
|
||||
},
|
||||
{
|
||||
"lower_ci": 961.617441776,
|
||||
"model": "pplx-70b-online",
|
||||
"rank": 83,
|
||||
"rating": 967.4367908935,
|
||||
"upper_ci": 972.8126406202
|
||||
},
|
||||
{
|
||||
"lower_ci": 959.5692952959,
|
||||
"model": "openchat-3.5",
|
||||
"rank": 84,
|
||||
"rating": 966.0510063396,
|
||||
"upper_ci": 971.8466170437
|
||||
},
|
||||
{
|
||||
"lower_ci": 956.7287406383,
|
||||
"model": "deepseek-llm-67b-chat",
|
||||
"rank": 85,
|
||||
"rating": 965.1147523375,
|
||||
"upper_ci": 971.4330625559
|
||||
},
|
||||
{
|
||||
"lower_ci": 956.3934469962,
|
||||
"model": "openhermes-2.5-mistral-7b",
|
||||
"rank": 86,
|
||||
"rating": 963.454411119,
|
||||
"upper_ci": 968.2145476373
|
||||
},
|
||||
{
|
||||
"lower_ci": 957.6795493747,
|
||||
"model": "mistral-7b-instruct-v0.2",
|
||||
"rank": 87,
|
||||
"rating": 961.195768793,
|
||||
"upper_ci": 964.3273838331
|
||||
},
|
||||
{
|
||||
"lower_ci": 955.9914563912,
|
||||
"model": "qwen1.5-7b-chat",
|
||||
"rank": 88,
|
||||
"rating": 959.7566395303,
|
||||
"upper_ci": 963.8434392791
|
||||
},
|
||||
{
|
||||
"lower_ci": 953.2396686903,
|
||||
"model": "phi-3-mini-4k-instruct-june-2024",
|
||||
"rank": 89,
|
||||
"rating": 958.8907631692,
|
||||
"upper_ci": 964.0199135738
|
||||
},
|
||||
{
|
||||
"lower_ci": 953.2564762147,
|
||||
"model": "gpt-3.5-turbo-1106",
|
||||
"rank": 90,
|
||||
"rating": 955.8557887224,
|
||||
"upper_ci": 959.2854635955
|
||||
},
|
||||
{
|
||||
"lower_ci": 951.2318268117,
|
||||
"model": "phi-3-mini-4k-instruct",
|
||||
"rank": 91,
|
||||
"rating": 955.5964299136,
|
||||
"upper_ci": 958.4767635868
|
||||
},
|
||||
{
|
||||
"lower_ci": 939.9598426545,
|
||||
"model": "dolphin-2.2.1-mistral-7b",
|
||||
"rank": 92,
|
||||
"rating": 952.2051167375,
|
||||
"upper_ci": 962.2676381777
|
||||
},
|
||||
{
|
||||
"lower_ci": 945.2431328588,
|
||||
"model": "solar-10.7b-instruct-v1.0",
|
||||
"rank": 93,
|
||||
"rating": 951.1224233106,
|
||||
"upper_ci": 958.6524028214
|
||||
},
|
||||
{
|
||||
"lower_ci": 948.6812080723,
|
||||
"model": "llama-2-13b-chat",
|
||||
"rank": 94,
|
||||
"rating": 951.0465511917,
|
||||
"upper_ci": 953.7276982632
|
||||
},
|
||||
{
|
||||
"lower_ci": 940.2487365654,
|
||||
"model": "wizardlm-13b",
|
||||
"rank": 95,
|
||||
"rating": 945.6170139554,
|
||||
"upper_ci": 952.3153173674
|
||||
},
|
||||
{
|
||||
"lower_ci": 937.3497020159,
|
||||
"model": "zephyr-7b-beta",
|
||||
"rank": 96,
|
||||
"rating": 941.7856240916,
|
||||
"upper_ci": 945.7193614882
|
||||
},
|
||||
{
|
||||
"lower_ci": 925.3514960917,
|
||||
"model": "mpt-30b-chat",
|
||||
"rank": 97,
|
||||
"rating": 933.4637447927,
|
||||
"upper_ci": 942.1936901157
|
||||
},
|
||||
{
|
||||
"lower_ci": 928.7552755121,
|
||||
"model": "pplx-7b-online",
|
||||
"rank": 98,
|
||||
"rating": 932.9740226673,
|
||||
"upper_ci": 940.2890736975
|
||||
},
|
||||
{
|
||||
"lower_ci": 924.8240945839,
|
||||
"model": "codellama-34b-instruct",
|
||||
"rank": 99,
|
||||
"rating": 931.25075507,
|
||||
"upper_ci": 936.6721012017
|
||||
},
|
||||
{
|
||||
"lower_ci": 919.9794338309,
|
||||
"model": "zephyr-7b-alpha",
|
||||
"rank": 100,
|
||||
"rating": 930.8024303718,
|
||||
"upper_ci": 940.3904190626
|
||||
},
|
||||
{
|
||||
"lower_ci": 927.6228696967,
|
||||
"model": "vicuna-13b",
|
||||
"rank": 101,
|
||||
"rating": 930.129806663,
|
||||
"upper_ci": 933.8873904378
|
||||
},
|
||||
{
|
||||
"lower_ci": 916.3079081804,
|
||||
"model": "codellama-70b-instruct",
|
||||
"rank": 102,
|
||||
"rating": 929.7441814853,
|
||||
"upper_ci": 939.4638589466
|
||||
},
|
||||
{
|
||||
"lower_ci": 919.0275736817,
|
||||
"model": "gemma-7b-it",
|
||||
"rank": 103,
|
||||
"rating": 925.6063141152,
|
||||
"upper_ci": 931.2321796245
|
||||
},
|
||||
{
|
||||
"lower_ci": 921.4335696267,
|
||||
"model": "llama-2-7b-chat",
|
||||
"rank": 104,
|
||||
"rating": 925.1226871516,
|
||||
"upper_ci": 927.4211774328
|
||||
},
|
||||
{
|
||||
"lower_ci": 919.914948524,
|
||||
"model": "phi-3-mini-128k-instruct",
|
||||
"rank": 105,
|
||||
"rating": 925.0941128748,
|
||||
"upper_ci": 928.953409783
|
||||
},
|
||||
{
|
||||
"lower_ci": 917.0649637172,
|
||||
"model": "qwen-14b-chat",
|
||||
"rank": 106,
|
||||
"rating": 923.0565707958,
|
||||
"upper_ci": 927.5147584562
|
||||
},
|
||||
{
|
||||
"lower_ci": 910.6316722798,
|
||||
"model": "falcon-180b-chat",
|
||||
"rank": 107,
|
||||
"rating": 922.8642075646,
|
||||
"upper_ci": 938.170426621
|
||||
},
|
||||
{
|
||||
"lower_ci": 912.8204058608,
|
||||
"model": "guanaco-33b",
|
||||
"rank": 108,
|
||||
"rating": 921.0201740806,
|
||||
"upper_ci": 929.2566479601
|
||||
},
|
||||
{
|
||||
"lower_ci": 903.2450471973,
|
||||
"model": "gemma-1.1-2b-it",
|
||||
"rank": 109,
|
||||
"rating": 908.7494526802,
|
||||
"upper_ci": 913.4741035764
|
||||
},
|
||||
{
|
||||
"lower_ci": 899.9846491667,
|
||||
"model": "stripedhyena-nous-7b",
|
||||
"rank": 110,
|
||||
"rating": 905.1534113535,
|
||||
"upper_ci": 912.9382997405
|
||||
},
|
||||
{
|
||||
"lower_ci": 899.9839053334,
|
||||
"model": "olmo-7b-instruct",
|
||||
"rank": 111,
|
||||
"rating": 903.629872537,
|
||||
"upper_ci": 908.2570306538
|
||||
},
|
||||
{
|
||||
"lower_ci": 890.8398870858,
|
||||
"model": "mistral-7b-instruct",
|
||||
"rank": 112,
|
||||
"rating": 897.2720855909,
|
||||
"upper_ci": 902.9292885859
|
||||
},
|
||||
{
|
||||
"lower_ci": 887.0047037055,
|
||||
"model": "vicuna-7b",
|
||||
"rank": 113,
|
||||
"rating": 893.825818387,
|
||||
"upper_ci": 899.7593423485
|
||||
},
|
||||
{
|
||||
"lower_ci": 887.0036385995,
|
||||
"model": "palm-2",
|
||||
"rank": 114,
|
||||
"rating": 891.3072880052,
|
||||
"upper_ci": 897.1963496269
|
||||
},
|
||||
{
|
||||
"lower_ci": 872.1578125039,
|
||||
"model": "gemma-2b-it",
|
||||
"rank": 115,
|
||||
"rating": 880.4830688196,
|
||||
"upper_ci": 884.0952002573
|
||||
},
|
||||
{
|
||||
"lower_ci": 868.4676861275,
|
||||
"model": "qwen1.5-4b-chat",
|
||||
"rank": 116,
|
||||
"rating": 877.0664166393,
|
||||
"upper_ci": 883.3511645246
|
||||
},
|
||||
{
|
||||
"lower_ci": 846.7609834499,
|
||||
"model": "koala-13b",
|
||||
"rank": 117,
|
||||
"rating": 852.5741703315,
|
||||
"upper_ci": 859.5370102373
|
||||
},
|
||||
{
|
||||
"lower_ci": 833.5366986653,
|
||||
"model": "chatglm3-6b",
|
||||
"rank": 118,
|
||||
"rating": 843.5467457427,
|
||||
"upper_ci": 853.6979083485
|
||||
},
|
||||
{
|
||||
"lower_ci": 808.672593832,
|
||||
"model": "gpt4all-13b-snoozy",
|
||||
"rank": 119,
|
||||
"rating": 821.0385947815,
|
||||
"upper_ci": 835.6716204673
|
||||
},
|
||||
{
|
||||
"lower_ci": 804.1197078456,
|
||||
"model": "chatglm2-6b",
|
||||
"rank": 120,
|
||||
"rating": 815.3737962537,
|
||||
"upper_ci": 821.0889058237
|
||||
},
|
||||
{
|
||||
"lower_ci": 805.0790145038,
|
||||
"model": "mpt-7b-chat",
|
||||
"rank": 121,
|
||||
"rating": 813.8798502443,
|
||||
"upper_ci": 823.374846381
|
||||
},
|
||||
{
|
||||
"lower_ci": 803.3055604626,
|
||||
"model": "RWKV-4-Raven-14B",
|
||||
"rank": 122,
|
||||
"rating": 810.0683748343,
|
||||
"upper_ci": 819.7203248755
|
||||
},
|
||||
{
|
||||
"lower_ci": 781.8369679949,
|
||||
"model": "alpaca-13b",
|
||||
"rank": 123,
|
||||
"rating": 789.0910768147,
|
||||
"upper_ci": 797.3012759671
|
||||
},
|
||||
{
|
||||
"lower_ci": 778.7113024193,
|
||||
"model": "oasst-pythia-12b",
|
||||
"rank": 124,
|
||||
"rating": 782.6327874047,
|
||||
"upper_ci": 789.0013974908
|
||||
},
|
||||
{
|
||||
"lower_ci": 760.5962982346,
|
||||
"model": "chatglm-6b",
|
||||
"rank": 125,
|
||||
"rating": 767.7699631784,
|
||||
"upper_ci": 776.2247501319
|
||||
},
|
||||
{
|
||||
"lower_ci": 749.711207841,
|
||||
"model": "fastchat-t5-3b",
|
||||
"rank": 126,
|
||||
"rating": 756.8214872887,
|
||||
"upper_ci": 762.6617679978
|
||||
},
|
||||
{
|
||||
"lower_ci": 723.1589751426,
|
||||
"model": "stablelm-tuned-alpha-7b",
|
||||
"rank": 127,
|
||||
"rating": 727.6986691281,
|
||||
"upper_ci": 737.7996673488
|
||||
},
|
||||
{
|
||||
"lower_ci": 701.8523006848,
|
||||
"model": "dolly-v2-12b",
|
||||
"rank": 128,
|
||||
"rating": 710.1023148692,
|
||||
"upper_ci": 717.2721178625
|
||||
},
|
||||
{
|
||||
"lower_ci": 679.2645194103,
|
||||
"model": "llama-13b",
|
||||
"rank": 129,
|
||||
"rating": 689.1631346499,
|
||||
"upper_ci": 694.5590720343
|
||||
}
|
||||
]
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 116 KiB |
+2224
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"artifacts": {
|
||||
"bradley_terry.json": {
|
||||
"bytes": 19827,
|
||||
"sha256": "4a861b80c69132fdb3e3d1cfdc9d962444d664623b956595e67798d141ee1c6b"
|
||||
},
|
||||
"leaderboard.png": {
|
||||
"bytes": 118255,
|
||||
"sha256": "386889f573198987400fef7c7803e0779b95e2fa35cb274d04071fed842c6499"
|
||||
},
|
||||
"leaderboard_animation.html": {
|
||||
"bytes": 54322,
|
||||
"sha256": "afb9b0426605dcc5accf9e1ea5067dd885ad9a9226884cc341b3430404ca433a"
|
||||
},
|
||||
"online_elo.json": {
|
||||
"bytes": 16885,
|
||||
"sha256": "41ec9a46bb37457a385d0c4b4036ff8ae3d1affcbae8b9b00689fef494f9f9a6"
|
||||
},
|
||||
"rating_history.json": {
|
||||
"bytes": 354362,
|
||||
"sha256": "3a5e3193cbbe50bf1de3a9e5d9702f2b594f449300d195674ed0da2a76f72574"
|
||||
},
|
||||
"rating_history.png": {
|
||||
"bytes": 193995,
|
||||
"sha256": "cb676a82b38a8876bc1346e58f72b6a9d72c8fa5e405a9d22610f6948382fba4"
|
||||
},
|
||||
"summary.json": {
|
||||
"bytes": 10441,
|
||||
"sha256": "6258914ebdb6be7686985e637903c929ed14535d8572a4b9d42b1256d5cdf261"
|
||||
},
|
||||
"win_rate_matrix.json": {
|
||||
"bytes": 37040,
|
||||
"sha256": "a6ac9580a8d0290d53adf16458ac777d72e214d0771cd0e72d1b2d8a4d86abd2"
|
||||
},
|
||||
"win_rate_matrix.png": {
|
||||
"bytes": 230883,
|
||||
"sha256": "db375436646096bfafdad458b53aba54ac156432cf9e8bb7458a2f733c9ae690"
|
||||
}
|
||||
},
|
||||
"experiment": "7-7",
|
||||
"gates": {
|
||||
"bradley_terry_official_method_completed": true,
|
||||
"chronological_online_elo_k4_completed": true,
|
||||
"d3_animation_saved": true,
|
||||
"millions_of_blind_votes_loaded": true,
|
||||
"monthly_history_saved": true,
|
||||
"official_public_arena_snapshot_hashed": true,
|
||||
"online_vs_official_method_rank_agreement_observed": true,
|
||||
"pairwise_empirical_and_predicted_matrix_saved": true,
|
||||
"static_visualizations_saved": true
|
||||
},
|
||||
"input": {
|
||||
"bytes": 2131976365,
|
||||
"filename": "arena_data.json",
|
||||
"sha256": "747c1c937dfa941d5a455a1fd70e2879d29642058da39b97996b977233b9bd1b",
|
||||
"url": "https://storage.googleapis.com/arena_external_data/public/clean_battle_20240814_public.json"
|
||||
},
|
||||
"official_complete": true,
|
||||
"runtime": {
|
||||
"numpy": "1.26.4",
|
||||
"pandas": "2.3.3",
|
||||
"platform": "macOS-26.3-arm64-arm-64bit",
|
||||
"python": "3.12.11"
|
||||
},
|
||||
"schema_version": 1,
|
||||
"sources": {
|
||||
"animation.py": "6839a3173a6a9ee8855901ca3c781fd91de13219b52ce6b285778ab89d926376",
|
||||
"bradley_terry.py": "030c716284017b14cff204ecd7036020c05554ae0eacf71af5da5c0963848b54",
|
||||
"optimized_elo.py": "f56b5b4aaba9cda11006bf01742be99fc07ee44ceeed5bd2dee5322a8ce08a57",
|
||||
"validation/run_experiment.py": "6c668081c4fbf04f9dbc84411a2a40391d7b00c76a90336542cc6a88623e991c",
|
||||
"validation/validate_evidence.py": "36fe448c94608cfc6a299f21224870357d109d999ba3391ed7d7c71fc1817e62"
|
||||
},
|
||||
"status": "passed"
|
||||
}
|
||||
@@ -0,0 +1,905 @@
|
||||
[
|
||||
{
|
||||
"matches": 55654,
|
||||
"model": "gemini-1.5-pro-api-0409-preview",
|
||||
"rank": 1,
|
||||
"rating": 1159.8203895968,
|
||||
"wins": 33738.5
|
||||
},
|
||||
{
|
||||
"matches": 20071,
|
||||
"model": "gemini-1.5-pro-exp-0801",
|
||||
"rank": 2,
|
||||
"rating": 1127.8877886511,
|
||||
"wins": 12139.5
|
||||
},
|
||||
{
|
||||
"matches": 14514,
|
||||
"model": "chatgpt-4o-latest",
|
||||
"rank": 3,
|
||||
"rating": 1126.6935165964,
|
||||
"wins": 8999.0
|
||||
},
|
||||
{
|
||||
"matches": 5655,
|
||||
"model": "gpt-3.5-turbo-0314",
|
||||
"rank": 4,
|
||||
"rating": 1103.9751202512,
|
||||
"wins": 3789.0
|
||||
},
|
||||
{
|
||||
"matches": 11827,
|
||||
"model": "bard-jan-24-gemini-pro",
|
||||
"rank": 5,
|
||||
"rating": 1094.2118678517,
|
||||
"wins": 7075.5
|
||||
},
|
||||
{
|
||||
"matches": 21172,
|
||||
"model": "claude-1",
|
||||
"rank": 6,
|
||||
"rating": 1092.7309868534,
|
||||
"wins": 12081.5
|
||||
},
|
||||
{
|
||||
"matches": 52155,
|
||||
"model": "gemini-advanced-0514",
|
||||
"rank": 7,
|
||||
"rating": 1087.5408038371,
|
||||
"wins": 30523.0
|
||||
},
|
||||
{
|
||||
"matches": 13604,
|
||||
"model": "llama-3.1-70b-instruct",
|
||||
"rank": 8,
|
||||
"rating": 1087.3529845908,
|
||||
"wins": 7058.5
|
||||
},
|
||||
{
|
||||
"matches": 77509,
|
||||
"model": "gpt-4o-2024-05-13",
|
||||
"rank": 9,
|
||||
"rating": 1086.7242770205,
|
||||
"wins": 46951.0
|
||||
},
|
||||
{
|
||||
"matches": 9761,
|
||||
"model": "gpt-4o-2024-08-06",
|
||||
"rank": 10,
|
||||
"rating": 1085.601246973,
|
||||
"wins": 5323.0
|
||||
},
|
||||
{
|
||||
"matches": 19370,
|
||||
"model": "gpt-4o-mini-2024-07-18",
|
||||
"rank": 11,
|
||||
"rating": 1079.187026245,
|
||||
"wins": 11082.0
|
||||
},
|
||||
{
|
||||
"matches": 47703,
|
||||
"model": "claude-3-5-sonnet-20240620",
|
||||
"rank": 12,
|
||||
"rating": 1079.0096726066,
|
||||
"wins": 27107.5
|
||||
},
|
||||
{
|
||||
"matches": 155944,
|
||||
"model": "claude-3-opus-20240229",
|
||||
"rank": 13,
|
||||
"rating": 1074.9078256001,
|
||||
"wins": 91432.5
|
||||
},
|
||||
{
|
||||
"matches": 11476,
|
||||
"model": "athene-70b-0725",
|
||||
"rank": 14,
|
||||
"rating": 1073.7607622933,
|
||||
"wins": 6098.0
|
||||
},
|
||||
{
|
||||
"matches": 18790,
|
||||
"model": "gemini-pro-dev-api",
|
||||
"rank": 15,
|
||||
"rating": 1072.3072263884,
|
||||
"wins": 8728.5
|
||||
},
|
||||
{
|
||||
"matches": 12774,
|
||||
"model": "claude-2.0",
|
||||
"rank": 16,
|
||||
"rating": 1069.530762176,
|
||||
"wins": 6814.5
|
||||
},
|
||||
{
|
||||
"matches": 10239,
|
||||
"model": "glm-4-0520",
|
||||
"rank": 17,
|
||||
"rating": 1065.2967580553,
|
||||
"wins": 5154.5
|
||||
},
|
||||
{
|
||||
"matches": 20669,
|
||||
"model": "nemotron-4-340b-instruct",
|
||||
"rank": 18,
|
||||
"rating": 1063.9797181947,
|
||||
"wins": 10567.5
|
||||
},
|
||||
{
|
||||
"matches": 51739,
|
||||
"model": "yi-large-preview",
|
||||
"rank": 19,
|
||||
"rating": 1060.8561460851,
|
||||
"wins": 28793.0
|
||||
},
|
||||
{
|
||||
"matches": 39631,
|
||||
"model": "llama-2-70b-chat",
|
||||
"rank": 20,
|
||||
"rating": 1060.6350445555,
|
||||
"wins": 18272.0
|
||||
},
|
||||
{
|
||||
"matches": 6974,
|
||||
"model": "reka-core-20240722",
|
||||
"rank": 21,
|
||||
"rating": 1059.9327626435,
|
||||
"wins": 3490.0
|
||||
},
|
||||
{
|
||||
"matches": 69418,
|
||||
"model": "gemini-1.5-pro-api-0514",
|
||||
"rank": 22,
|
||||
"rating": 1059.8892020659,
|
||||
"wins": 39302.5
|
||||
},
|
||||
{
|
||||
"matches": 6565,
|
||||
"model": "gemini-pro",
|
||||
"rank": 23,
|
||||
"rating": 1059.2020149016,
|
||||
"wins": 3028.0
|
||||
},
|
||||
{
|
||||
"matches": 18897,
|
||||
"model": "llama-3.1-405b-instruct",
|
||||
"rank": 24,
|
||||
"rating": 1058.4003562569,
|
||||
"wins": 10361.5
|
||||
},
|
||||
{
|
||||
"matches": 11517,
|
||||
"model": "mistral-large-2407",
|
||||
"rank": 25,
|
||||
"rating": 1058.1277726399,
|
||||
"wins": 6038.5
|
||||
},
|
||||
{
|
||||
"matches": 84505,
|
||||
"model": "gpt-4-turbo-2024-04-09",
|
||||
"rank": 26,
|
||||
"rating": 1054.9014485987,
|
||||
"wins": 48844.5
|
||||
},
|
||||
{
|
||||
"matches": 92488,
|
||||
"model": "gpt-4-1106-preview",
|
||||
"rank": 27,
|
||||
"rating": 1054.2069148591,
|
||||
"wins": 57496.5
|
||||
},
|
||||
{
|
||||
"matches": 38959,
|
||||
"model": "gpt-3.5-turbo-0613",
|
||||
"rank": 28,
|
||||
"rating": 1052.6707187439,
|
||||
"wins": 19308.0
|
||||
},
|
||||
{
|
||||
"matches": 85867,
|
||||
"model": "gpt-4-0125-preview",
|
||||
"rank": 29,
|
||||
"rating": 1050.1218586343,
|
||||
"wins": 50802.5
|
||||
},
|
||||
{
|
||||
"matches": 7579,
|
||||
"model": "glm-4-0116",
|
||||
"rank": 30,
|
||||
"rating": 1049.1894798578,
|
||||
"wins": 3566.0
|
||||
},
|
||||
{
|
||||
"matches": 4858,
|
||||
"model": "zephyr-orpo-141b-A35b-v0.1",
|
||||
"rank": 31,
|
||||
"rating": 1045.2216087838,
|
||||
"wins": 2116.5
|
||||
},
|
||||
{
|
||||
"matches": 25748,
|
||||
"model": "qwen-max-0428",
|
||||
"rank": 32,
|
||||
"rating": 1044.8335252342,
|
||||
"wins": 12269.5
|
||||
},
|
||||
{
|
||||
"matches": 20633,
|
||||
"model": "claude-instant-1",
|
||||
"rank": 33,
|
||||
"rating": 1044.7562986355,
|
||||
"wins": 11178.0
|
||||
},
|
||||
{
|
||||
"matches": 16672,
|
||||
"model": "yi-large",
|
||||
"rank": 34,
|
||||
"rating": 1044.7374546104,
|
||||
"wins": 8533.5
|
||||
},
|
||||
{
|
||||
"matches": 3997,
|
||||
"model": "deepseek-coder-v2-0724",
|
||||
"rank": 35,
|
||||
"rating": 1043.8935002553,
|
||||
"wins": 1883.5
|
||||
},
|
||||
{
|
||||
"matches": 16165,
|
||||
"model": "deepseek-v2-api-0628",
|
||||
"rank": 36,
|
||||
"rating": 1042.4852039574,
|
||||
"wins": 7994.5
|
||||
},
|
||||
{
|
||||
"matches": 56129,
|
||||
"model": "gemini-1.5-flash-api-0514",
|
||||
"rank": 37,
|
||||
"rating": 1041.3766839839,
|
||||
"wins": 29778.0
|
||||
},
|
||||
{
|
||||
"matches": 161827,
|
||||
"model": "llama-3-70b-instruct",
|
||||
"rank": 38,
|
||||
"rating": 1040.8719399645,
|
||||
"wins": 84977.0
|
||||
},
|
||||
{
|
||||
"matches": 80911,
|
||||
"model": "command-r-plus",
|
||||
"rank": 39,
|
||||
"rating": 1034.8822379837,
|
||||
"wins": 40967.0
|
||||
},
|
||||
{
|
||||
"matches": 55979,
|
||||
"model": "gpt-4-0314",
|
||||
"rank": 40,
|
||||
"rating": 1033.6412751608,
|
||||
"wins": 31098.0
|
||||
},
|
||||
{
|
||||
"matches": 37702,
|
||||
"model": "claude-2.1",
|
||||
"rank": 41,
|
||||
"rating": 1032.8143823524,
|
||||
"wins": 16848.0
|
||||
},
|
||||
{
|
||||
"matches": 8391,
|
||||
"model": "wizardlm-70b",
|
||||
"rank": 42,
|
||||
"rating": 1032.5111769139,
|
||||
"wins": 4325.0
|
||||
},
|
||||
{
|
||||
"matches": 27838,
|
||||
"model": "gemma-2-27b-it",
|
||||
"rank": 43,
|
||||
"rating": 1030.7976774989,
|
||||
"wins": 13981.5
|
||||
},
|
||||
{
|
||||
"matches": 1714,
|
||||
"model": "dolphin-2.2.1-mistral-7b",
|
||||
"rank": 44,
|
||||
"rating": 1029.9575607586,
|
||||
"wins": 738.5
|
||||
},
|
||||
{
|
||||
"matches": 3003,
|
||||
"model": "guanaco-33b",
|
||||
"rank": 45,
|
||||
"rating": 1027.2582838587,
|
||||
"wins": 1552.0
|
||||
},
|
||||
{
|
||||
"matches": 3843,
|
||||
"model": "nous-hermes-2-mixtral-8x7b-dpo",
|
||||
"rank": 46,
|
||||
"rating": 1025.2145335894,
|
||||
"wins": 1697.5
|
||||
},
|
||||
{
|
||||
"matches": 7192,
|
||||
"model": "wizardlm-13b",
|
||||
"rank": 47,
|
||||
"rating": 1024.793903105,
|
||||
"wins": 3690.0
|
||||
},
|
||||
{
|
||||
"matches": 2650,
|
||||
"model": "mpt-30b-chat",
|
||||
"rank": 48,
|
||||
"rating": 1024.0563241095,
|
||||
"wins": 1351.5
|
||||
},
|
||||
{
|
||||
"matches": 27508,
|
||||
"model": "qwen1.5-110b-chat",
|
||||
"rank": 49,
|
||||
"rating": 1023.7761579902,
|
||||
"wins": 12612.0
|
||||
},
|
||||
{
|
||||
"matches": 113106,
|
||||
"model": "claude-3-sonnet-20240229",
|
||||
"rank": 50,
|
||||
"rating": 1023.6338810846,
|
||||
"wins": 60284.5
|
||||
},
|
||||
{
|
||||
"matches": 12377,
|
||||
"model": "mistral-next",
|
||||
"rank": 51,
|
||||
"rating": 1021.9755255724,
|
||||
"wins": 5952.5
|
||||
},
|
||||
{
|
||||
"matches": 15801,
|
||||
"model": "deepseek-coder-v2",
|
||||
"rank": 52,
|
||||
"rating": 1021.6778455715,
|
||||
"wins": 7243.0
|
||||
},
|
||||
{
|
||||
"matches": 16056,
|
||||
"model": "reka-flash-21b-20240226-online",
|
||||
"rank": 53,
|
||||
"rating": 1020.8238893282,
|
||||
"wins": 7554.5
|
||||
},
|
||||
{
|
||||
"matches": 16667,
|
||||
"model": "starling-lm-7b-beta",
|
||||
"rank": 54,
|
||||
"rating": 1020.1246920766,
|
||||
"wins": 7626.0
|
||||
},
|
||||
{
|
||||
"matches": 3636,
|
||||
"model": "llama2-70b-steerlm-chat",
|
||||
"rank": 55,
|
||||
"rating": 1018.5882277478,
|
||||
"wins": 1587.5
|
||||
},
|
||||
{
|
||||
"matches": 35555,
|
||||
"model": "mistral-medium",
|
||||
"rank": 56,
|
||||
"rating": 1018.0043978294,
|
||||
"wins": 17731.5
|
||||
},
|
||||
{
|
||||
"matches": 19740,
|
||||
"model": "llama-2-13b-chat",
|
||||
"rank": 57,
|
||||
"rating": 1016.6652779795,
|
||||
"wins": 8935.5
|
||||
},
|
||||
{
|
||||
"matches": 6660,
|
||||
"model": "tulu-2-dpo-70b",
|
||||
"rank": 58,
|
||||
"rating": 1014.6804751782,
|
||||
"wins": 3231.0
|
||||
},
|
||||
{
|
||||
"matches": 62688,
|
||||
"model": "reka-core-20240501",
|
||||
"rank": 59,
|
||||
"rating": 1013.8222528866,
|
||||
"wins": 31198.0
|
||||
},
|
||||
{
|
||||
"matches": 89617,
|
||||
"model": "gpt-4-0613",
|
||||
"rank": 60,
|
||||
"rating": 1013.3276942754,
|
||||
"wins": 43878.0
|
||||
},
|
||||
{
|
||||
"matches": 4981,
|
||||
"model": "deepseek-llm-67b-chat",
|
||||
"rank": 61,
|
||||
"rating": 1011.6783077357,
|
||||
"wins": 2017.0
|
||||
},
|
||||
{
|
||||
"matches": 4291,
|
||||
"model": "solar-10.7b-instruct-v1.0",
|
||||
"rank": 62,
|
||||
"rating": 1011.6055213958,
|
||||
"wins": 1814.0
|
||||
},
|
||||
{
|
||||
"matches": 12987,
|
||||
"model": "openchat-3.5-0106",
|
||||
"rank": 63,
|
||||
"rating": 1009.5145969972,
|
||||
"wins": 5974.0
|
||||
},
|
||||
{
|
||||
"matches": 7156,
|
||||
"model": "reka-flash-20240722",
|
||||
"rank": 64,
|
||||
"rating": 1003.5048766039,
|
||||
"wins": 3290.0
|
||||
},
|
||||
{
|
||||
"matches": 25248,
|
||||
"model": "gemma-2-9b-it",
|
||||
"rank": 65,
|
||||
"rating": 1003.026164166,
|
||||
"wins": 11661.5
|
||||
},
|
||||
{
|
||||
"matches": 12671,
|
||||
"model": "llama-3.1-8b-instruct",
|
||||
"rank": 66,
|
||||
"rating": 1002.2631876345,
|
||||
"wins": 5121.0
|
||||
},
|
||||
{
|
||||
"matches": 8117,
|
||||
"model": "openchat-3.5",
|
||||
"rank": 67,
|
||||
"rating": 1001.939993772,
|
||||
"wins": 3783.0
|
||||
},
|
||||
{
|
||||
"matches": 6338,
|
||||
"model": "pplx-7b-online",
|
||||
"rank": 68,
|
||||
"rating": 1001.6927968751,
|
||||
"wins": 2572.0
|
||||
},
|
||||
{
|
||||
"matches": 40643,
|
||||
"model": "qwen1.5-72b-chat",
|
||||
"rank": 69,
|
||||
"rating": 1000.9425509065,
|
||||
"wins": 19941.0
|
||||
},
|
||||
{
|
||||
"matches": 1815,
|
||||
"model": "zephyr-7b-alpha",
|
||||
"rank": 70,
|
||||
"rating": 999.8845726251,
|
||||
"wins": 855.0
|
||||
},
|
||||
{
|
||||
"matches": 109888,
|
||||
"model": "claude-3-haiku-20240307",
|
||||
"rank": 71,
|
||||
"rating": 999.7072031054,
|
||||
"wins": 55074.0
|
||||
},
|
||||
{
|
||||
"matches": 10422,
|
||||
"model": "starling-lm-7b-alpha",
|
||||
"rank": 72,
|
||||
"rating": 998.0936577805,
|
||||
"wins": 4817.0
|
||||
},
|
||||
{
|
||||
"matches": 25792,
|
||||
"model": "reka-flash-21b-20240226",
|
||||
"rank": 73,
|
||||
"rating": 992.8107482264,
|
||||
"wins": 11891.0
|
||||
},
|
||||
{
|
||||
"matches": 64938,
|
||||
"model": "mistral-large-2402",
|
||||
"rank": 74,
|
||||
"rating": 992.5035561283,
|
||||
"wins": 30754.5
|
||||
},
|
||||
{
|
||||
"matches": 17026,
|
||||
"model": "gpt-3.5-turbo-1106",
|
||||
"rank": 75,
|
||||
"rating": 990.5828502496,
|
||||
"wins": 7026.0
|
||||
},
|
||||
{
|
||||
"matches": 4864,
|
||||
"model": "qwen1.5-7b-chat",
|
||||
"rank": 76,
|
||||
"rating": 990.5290268684,
|
||||
"wins": 1903.5
|
||||
},
|
||||
{
|
||||
"matches": 20477,
|
||||
"model": "reka-flash-preview-20240611",
|
||||
"rank": 77,
|
||||
"rating": 990.4289024537,
|
||||
"wins": 9003.5
|
||||
},
|
||||
{
|
||||
"matches": 25156,
|
||||
"model": "yi-1.5-34b-chat",
|
||||
"rank": 78,
|
||||
"rating": 988.8079557694,
|
||||
"wins": 11066.0
|
||||
},
|
||||
{
|
||||
"matches": 5091,
|
||||
"model": "openhermes-2.5-mistral-7b",
|
||||
"rank": 79,
|
||||
"rating": 988.507956639,
|
||||
"wins": 2379.0
|
||||
},
|
||||
{
|
||||
"matches": 7508,
|
||||
"model": "codellama-34b-instruct",
|
||||
"rank": 80,
|
||||
"rating": 986.9963372601,
|
||||
"wins": 3192.5
|
||||
},
|
||||
{
|
||||
"matches": 18666,
|
||||
"model": "qwen1.5-14b-chat",
|
||||
"rank": 81,
|
||||
"rating": 985.79676379,
|
||||
"wins": 8265.0
|
||||
},
|
||||
{
|
||||
"matches": 15946,
|
||||
"model": "yi-34b-chat",
|
||||
"rank": 82,
|
||||
"rating": 984.9789753527,
|
||||
"wins": 7423.5
|
||||
},
|
||||
{
|
||||
"matches": 6890,
|
||||
"model": "pplx-70b-online",
|
||||
"rank": 83,
|
||||
"rating": 984.0082457269,
|
||||
"wins": 3118.0
|
||||
},
|
||||
{
|
||||
"matches": 34410,
|
||||
"model": "qwen2-72b-instruct",
|
||||
"rank": 84,
|
||||
"rating": 983.7228839439,
|
||||
"wins": 16578.5
|
||||
},
|
||||
{
|
||||
"matches": 33736,
|
||||
"model": "dbrx-instruct-preview",
|
||||
"rank": 85,
|
||||
"rating": 983.0655232047,
|
||||
"wins": 14350.5
|
||||
},
|
||||
{
|
||||
"matches": 107100,
|
||||
"model": "llama-3-8b-instruct",
|
||||
"rank": 86,
|
||||
"rating": 978.2506897907,
|
||||
"wins": 49020.5
|
||||
},
|
||||
{
|
||||
"matches": 1325,
|
||||
"model": "falcon-180b-chat",
|
||||
"rank": 87,
|
||||
"rating": 976.0053410687,
|
||||
"wins": 580.0
|
||||
},
|
||||
{
|
||||
"matches": 8745,
|
||||
"model": "palm-2",
|
||||
"rank": 88,
|
||||
"rating": 975.4724123669,
|
||||
"wins": 3851.5
|
||||
},
|
||||
{
|
||||
"matches": 5070,
|
||||
"model": "qwen-14b-chat",
|
||||
"rank": 89,
|
||||
"rating": 974.4393986925,
|
||||
"wins": 2262.5
|
||||
},
|
||||
{
|
||||
"matches": 5270,
|
||||
"model": "stripedhyena-nous-7b",
|
||||
"rank": 90,
|
||||
"rating": 972.5214538963,
|
||||
"wins": 2001.0
|
||||
},
|
||||
{
|
||||
"matches": 22783,
|
||||
"model": "qwen1.5-32b-chat",
|
||||
"rank": 91,
|
||||
"rating": 970.5573357551,
|
||||
"wins": 10038.5
|
||||
},
|
||||
{
|
||||
"matches": 56373,
|
||||
"model": "command-r",
|
||||
"rank": 92,
|
||||
"rating": 968.3429828775,
|
||||
"wins": 25830.5
|
||||
},
|
||||
{
|
||||
"matches": 9178,
|
||||
"model": "gemma-7b-it",
|
||||
"rank": 93,
|
||||
"rating": 966.2526965022,
|
||||
"wins": 3447.5
|
||||
},
|
||||
{
|
||||
"matches": 11330,
|
||||
"model": "zephyr-7b-beta",
|
||||
"rank": 94,
|
||||
"rating": 966.1969243012,
|
||||
"wins": 5268.0
|
||||
},
|
||||
{
|
||||
"matches": 50256,
|
||||
"model": "mixtral-8x22b-instruct-v0.1",
|
||||
"rank": 95,
|
||||
"rating": 964.4141739849,
|
||||
"wins": 22236.0
|
||||
},
|
||||
{
|
||||
"matches": 7022,
|
||||
"model": "vicuna-7b",
|
||||
"rank": 96,
|
||||
"rating": 963.6362226151,
|
||||
"wins": 3187.5
|
||||
},
|
||||
{
|
||||
"matches": 34197,
|
||||
"model": "snowflake-arctic-instruct",
|
||||
"rank": 97,
|
||||
"rating": 961.9084215198,
|
||||
"wins": 13326.5
|
||||
},
|
||||
{
|
||||
"matches": 22938,
|
||||
"model": "vicuna-33b",
|
||||
"rank": 98,
|
||||
"rating": 959.3823284501,
|
||||
"wins": 10924.5
|
||||
},
|
||||
{
|
||||
"matches": 10509,
|
||||
"model": "gemma-2-2b-it",
|
||||
"rank": 99,
|
||||
"rating": 959.2713799268,
|
||||
"wins": 3731.5
|
||||
},
|
||||
{
|
||||
"matches": 7038,
|
||||
"model": "koala-13b",
|
||||
"rank": 100,
|
||||
"rating": 954.395027749,
|
||||
"wins": 3380.0
|
||||
},
|
||||
{
|
||||
"matches": 68914,
|
||||
"model": "gpt-3.5-turbo-0125",
|
||||
"rank": 101,
|
||||
"rating": 951.9181205239,
|
||||
"wins": 29165.5
|
||||
},
|
||||
{
|
||||
"matches": 9145,
|
||||
"model": "mistral-7b-instruct",
|
||||
"rank": 102,
|
||||
"rating": 948.891187234,
|
||||
"wins": 3685.5
|
||||
},
|
||||
{
|
||||
"matches": 14559,
|
||||
"model": "llama-2-7b-chat",
|
||||
"rank": 103,
|
||||
"rating": 948.4212635676,
|
||||
"wins": 6212.0
|
||||
},
|
||||
{
|
||||
"matches": 20074,
|
||||
"model": "mistral-7b-instruct-v0.2",
|
||||
"rank": 104,
|
||||
"rating": 945.8420359878,
|
||||
"wins": 8282.5
|
||||
},
|
||||
{
|
||||
"matches": 25095,
|
||||
"model": "gemma-1.1-7b-it",
|
||||
"rank": 105,
|
||||
"rating": 941.8141470663,
|
||||
"wins": 9687.0
|
||||
},
|
||||
{
|
||||
"matches": 1787,
|
||||
"model": "gpt4all-13b-snoozy",
|
||||
"rank": 106,
|
||||
"rating": 938.6958755482,
|
||||
"wins": 680.0
|
||||
},
|
||||
{
|
||||
"matches": 18505,
|
||||
"model": "phi-3-small-8k-instruct",
|
||||
"rank": 107,
|
||||
"rating": 930.6944270029,
|
||||
"wins": 7100.0
|
||||
},
|
||||
{
|
||||
"matches": 6496,
|
||||
"model": "olmo-7b-instruct",
|
||||
"rank": 108,
|
||||
"rating": 928.8306135323,
|
||||
"wins": 2157.0
|
||||
},
|
||||
{
|
||||
"matches": 21129,
|
||||
"model": "phi-3-mini-4k-instruct",
|
||||
"rank": 109,
|
||||
"rating": 928.4371869598,
|
||||
"wins": 7094.0
|
||||
},
|
||||
{
|
||||
"matches": 21617,
|
||||
"model": "phi-3-mini-128k-instruct",
|
||||
"rank": 110,
|
||||
"rating": 928.3336295806,
|
||||
"wins": 7094.5
|
||||
},
|
||||
{
|
||||
"matches": 4938,
|
||||
"model": "RWKV-4-Raven-14B",
|
||||
"rank": 111,
|
||||
"rating": 927.486785137,
|
||||
"wins": 1965.5
|
||||
},
|
||||
{
|
||||
"matches": 19794,
|
||||
"model": "vicuna-13b",
|
||||
"rank": 112,
|
||||
"rating": 927.1632498042,
|
||||
"wins": 9553.5
|
||||
},
|
||||
{
|
||||
"matches": 1193,
|
||||
"model": "codellama-70b-instruct",
|
||||
"rank": 113,
|
||||
"rating": 926.7141337337,
|
||||
"wins": 411.5
|
||||
},
|
||||
{
|
||||
"matches": 4018,
|
||||
"model": "mpt-7b-chat",
|
||||
"rank": 114,
|
||||
"rating": 922.0268289601,
|
||||
"wins": 1573.5
|
||||
},
|
||||
{
|
||||
"matches": 76064,
|
||||
"model": "mixtral-8x7b-instruct-v0.1",
|
||||
"rank": 115,
|
||||
"rating": 920.8542542685,
|
||||
"wins": 33437.0
|
||||
},
|
||||
{
|
||||
"matches": 22427,
|
||||
"model": "phi-3-medium-4k-instruct",
|
||||
"rank": 116,
|
||||
"rating": 919.7045738753,
|
||||
"wins": 8854.0
|
||||
},
|
||||
{
|
||||
"matches": 4921,
|
||||
"model": "gemma-2b-it",
|
||||
"rank": 117,
|
||||
"rating": 918.581510353,
|
||||
"wins": 1513.0
|
||||
},
|
||||
{
|
||||
"matches": 10484,
|
||||
"model": "phi-3-mini-4k-instruct-june-2024",
|
||||
"rank": 118,
|
||||
"rating": 913.3885931532,
|
||||
"wins": 3097.5
|
||||
},
|
||||
{
|
||||
"matches": 7811,
|
||||
"model": "qwen1.5-4b-chat",
|
||||
"rank": 119,
|
||||
"rating": 910.8121423587,
|
||||
"wins": 2364.5
|
||||
},
|
||||
{
|
||||
"matches": 4995,
|
||||
"model": "chatglm-6b",
|
||||
"rank": 120,
|
||||
"rating": 908.8611775605,
|
||||
"wins": 1834.5
|
||||
},
|
||||
{
|
||||
"matches": 5874,
|
||||
"model": "alpaca-13b",
|
||||
"rank": 121,
|
||||
"rating": 904.2619553966,
|
||||
"wins": 2330.0
|
||||
},
|
||||
{
|
||||
"matches": 11363,
|
||||
"model": "gemma-1.1-2b-it",
|
||||
"rank": 122,
|
||||
"rating": 892.2955737397,
|
||||
"wins": 3282.0
|
||||
},
|
||||
{
|
||||
"matches": 2707,
|
||||
"model": "chatglm2-6b",
|
||||
"rank": 123,
|
||||
"rating": 888.2580854124,
|
||||
"wins": 808.0
|
||||
},
|
||||
{
|
||||
"matches": 3334,
|
||||
"model": "stablelm-tuned-alpha-7b",
|
||||
"rank": 124,
|
||||
"rating": 882.7115402984,
|
||||
"wins": 1141.5
|
||||
},
|
||||
{
|
||||
"matches": 4763,
|
||||
"model": "chatglm3-6b",
|
||||
"rank": 125,
|
||||
"rating": 867.498613261,
|
||||
"wins": 1454.0
|
||||
},
|
||||
{
|
||||
"matches": 6382,
|
||||
"model": "oasst-pythia-12b",
|
||||
"rank": 126,
|
||||
"rating": 865.3424632797,
|
||||
"wins": 2425.0
|
||||
},
|
||||
{
|
||||
"matches": 2443,
|
||||
"model": "llama-13b",
|
||||
"rank": 127,
|
||||
"rating": 852.9506118392,
|
||||
"wins": 745.5
|
||||
},
|
||||
{
|
||||
"matches": 4304,
|
||||
"model": "fastchat-t5-3b",
|
||||
"rank": 128,
|
||||
"rating": 847.3254654092,
|
||||
"wins": 1465.0
|
||||
},
|
||||
{
|
||||
"matches": 3484,
|
||||
"model": "dolly-v2-12b",
|
||||
"rank": 129,
|
||||
"rating": 834.3876685486,
|
||||
"wins": 1099.5
|
||||
}
|
||||
]
|
||||
+17546
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 189 KiB |
@@ -0,0 +1,387 @@
|
||||
{
|
||||
"dataset": {
|
||||
"accepted_records": 1670250,
|
||||
"bounded_test_run": false,
|
||||
"bytes": 2131976365,
|
||||
"end_utc": "2024-08-14T19:52:53.409200+00:00",
|
||||
"load_filter_seconds": 15.133,
|
||||
"model_count": 129,
|
||||
"outcomes": {
|
||||
"model_a": 543065,
|
||||
"model_b": 550810,
|
||||
"tie": 281121,
|
||||
"tie (bothbad)": 295254
|
||||
},
|
||||
"path_recorded_as": "arena_data.json",
|
||||
"sha256": "747c1c937dfa941d5a455a1fd70e2879d29642058da39b97996b977233b9bd1b",
|
||||
"source_records": 1799991,
|
||||
"start_utc": "2023-04-24T15:53:11.132200+00:00",
|
||||
"url": "https://storage.googleapis.com/arena_external_data/public/clean_battle_20240814_public.json"
|
||||
},
|
||||
"experiment": "7-7",
|
||||
"gates": {
|
||||
"bradley_terry_official_method_completed": true,
|
||||
"chronological_online_elo_k4_completed": true,
|
||||
"d3_animation_saved": true,
|
||||
"millions_of_blind_votes_loaded": true,
|
||||
"monthly_history_saved": true,
|
||||
"official_public_arena_snapshot_hashed": true,
|
||||
"online_vs_official_method_rank_agreement_observed": true,
|
||||
"pairwise_empirical_and_predicted_matrix_saved": true,
|
||||
"static_visualizations_saved": true
|
||||
},
|
||||
"generated_at_utc": "2026-07-31T09:59:17.815099+00:00",
|
||||
"official_complete": true,
|
||||
"protocol": {
|
||||
"bootstrap_random_seed": 0,
|
||||
"bootstrap_rounds": 20,
|
||||
"history_interval": "monthly cumulative snapshots",
|
||||
"official_method": "Bradley-Terry maximum-likelihood reconstruction",
|
||||
"online_elo": "initial=1000, K=4, stable chronological order"
|
||||
},
|
||||
"results": {
|
||||
"official_method_top_20": [
|
||||
{
|
||||
"lower_ci": 1199.4356201692,
|
||||
"model": "chatgpt-4o-latest",
|
||||
"rank": 1,
|
||||
"rating": 1202.8722324768,
|
||||
"upper_ci": 1207.3805726125
|
||||
},
|
||||
{
|
||||
"lower_ci": 1182.767114327,
|
||||
"model": "gemini-1.5-pro-exp-0801",
|
||||
"rank": 2,
|
||||
"rating": 1187.3727762743,
|
||||
"upper_ci": 1191.7564879767
|
||||
},
|
||||
{
|
||||
"lower_ci": 1173.1270556478,
|
||||
"model": "gpt-4o-2024-05-13",
|
||||
"rank": 3,
|
||||
"rating": 1174.5475812185,
|
||||
"upper_ci": 1177.0403252386
|
||||
},
|
||||
{
|
||||
"lower_ci": 1161.1978031325,
|
||||
"model": "gpt-4o-mini-2024-07-18",
|
||||
"rank": 4,
|
||||
"rating": 1162.9712796813,
|
||||
"upper_ci": 1166.568334134
|
||||
},
|
||||
{
|
||||
"lower_ci": 1156.0224735749,
|
||||
"model": "claude-3-5-sonnet-20240620",
|
||||
"rank": 5,
|
||||
"rating": 1159.6692177384,
|
||||
"upper_ci": 1161.8458744744
|
||||
},
|
||||
{
|
||||
"lower_ci": 1153.6447817793,
|
||||
"model": "gemini-advanced-0514",
|
||||
"rank": 6,
|
||||
"rating": 1155.6305094344,
|
||||
"upper_ci": 1157.1030226933
|
||||
},
|
||||
{
|
||||
"lower_ci": 1150.4614200682,
|
||||
"model": "llama-3.1-405b-instruct",
|
||||
"rank": 7,
|
||||
"rating": 1153.3950109278,
|
||||
"upper_ci": 1157.0848375454
|
||||
},
|
||||
{
|
||||
"lower_ci": 1146.348878889,
|
||||
"model": "gpt-4o-2024-08-06",
|
||||
"rank": 8,
|
||||
"rating": 1150.7428051987,
|
||||
"upper_ci": 1156.3245456986
|
||||
},
|
||||
{
|
||||
"lower_ci": 1146.7772301944,
|
||||
"model": "gemini-1.5-pro-api-0514",
|
||||
"rank": 9,
|
||||
"rating": 1149.1265569601,
|
||||
"upper_ci": 1151.5010539806
|
||||
},
|
||||
{
|
||||
"lower_ci": 1143.6682527667,
|
||||
"model": "gemini-1.5-pro-api-0409-preview",
|
||||
"rank": 10,
|
||||
"rating": 1146.1756037638,
|
||||
"upper_ci": 1147.8895040945
|
||||
},
|
||||
{
|
||||
"lower_ci": 1142.3990049162,
|
||||
"model": "gpt-4-turbo-2024-04-09",
|
||||
"rank": 11,
|
||||
"rating": 1144.9187340632,
|
||||
"upper_ci": 1146.8362809167
|
||||
},
|
||||
{
|
||||
"lower_ci": 1138.1303324947,
|
||||
"model": "gpt-4-1106-preview",
|
||||
"rank": 12,
|
||||
"rating": 1139.5004505155,
|
||||
"upper_ci": 1140.97644635
|
||||
},
|
||||
{
|
||||
"lower_ci": 1136.3852091186,
|
||||
"model": "mistral-large-2407",
|
||||
"rank": 13,
|
||||
"rating": 1139.3566231011,
|
||||
"upper_ci": 1143.8960727747
|
||||
},
|
||||
{
|
||||
"lower_ci": 1135.0624903381,
|
||||
"model": "athene-70b-0725",
|
||||
"rank": 14,
|
||||
"rating": 1138.2016223593,
|
||||
"upper_ci": 1141.397916447
|
||||
},
|
||||
{
|
||||
"lower_ci": 1135.0278139027,
|
||||
"model": "claude-3-opus-20240229",
|
||||
"rank": 15,
|
||||
"rating": 1136.5739464324,
|
||||
"upper_ci": 1137.5663039742
|
||||
},
|
||||
{
|
||||
"lower_ci": 1131.7219814044,
|
||||
"model": "llama-3.1-70b-instruct",
|
||||
"rank": 16,
|
||||
"rating": 1134.8942150397,
|
||||
"upper_ci": 1139.3028524197
|
||||
},
|
||||
{
|
||||
"lower_ci": 1132.1800230236,
|
||||
"model": "gpt-4-0125-preview",
|
||||
"rank": 17,
|
||||
"rating": 1133.974390087,
|
||||
"upper_ci": 1135.7552442263
|
||||
},
|
||||
{
|
||||
"lower_ci": 1127.3300289872,
|
||||
"model": "yi-large-preview",
|
||||
"rank": 18,
|
||||
"rating": 1128.1692229165,
|
||||
"upper_ci": 1131.371743138
|
||||
},
|
||||
{
|
||||
"lower_ci": 1111.850158929,
|
||||
"model": "reka-core-20240722",
|
||||
"rank": 19,
|
||||
"rating": 1117.1464671073,
|
||||
"upper_ci": 1121.9537941313
|
||||
},
|
||||
{
|
||||
"lower_ci": 1114.0244965884,
|
||||
"model": "gemini-1.5-flash-api-0514",
|
||||
"rank": 20,
|
||||
"rating": 1116.6161359294,
|
||||
"upper_ci": 1118.5223962538
|
||||
}
|
||||
],
|
||||
"online_top_20": [
|
||||
{
|
||||
"matches": 55654,
|
||||
"model": "gemini-1.5-pro-api-0409-preview",
|
||||
"rank": 1,
|
||||
"rating": 1159.8203895968,
|
||||
"wins": 33738.5
|
||||
},
|
||||
{
|
||||
"matches": 20071,
|
||||
"model": "gemini-1.5-pro-exp-0801",
|
||||
"rank": 2,
|
||||
"rating": 1127.8877886511,
|
||||
"wins": 12139.5
|
||||
},
|
||||
{
|
||||
"matches": 14514,
|
||||
"model": "chatgpt-4o-latest",
|
||||
"rank": 3,
|
||||
"rating": 1126.6935165964,
|
||||
"wins": 8999.0
|
||||
},
|
||||
{
|
||||
"matches": 5655,
|
||||
"model": "gpt-3.5-turbo-0314",
|
||||
"rank": 4,
|
||||
"rating": 1103.9751202512,
|
||||
"wins": 3789.0
|
||||
},
|
||||
{
|
||||
"matches": 11827,
|
||||
"model": "bard-jan-24-gemini-pro",
|
||||
"rank": 5,
|
||||
"rating": 1094.2118678517,
|
||||
"wins": 7075.5
|
||||
},
|
||||
{
|
||||
"matches": 21172,
|
||||
"model": "claude-1",
|
||||
"rank": 6,
|
||||
"rating": 1092.7309868534,
|
||||
"wins": 12081.5
|
||||
},
|
||||
{
|
||||
"matches": 52155,
|
||||
"model": "gemini-advanced-0514",
|
||||
"rank": 7,
|
||||
"rating": 1087.5408038371,
|
||||
"wins": 30523.0
|
||||
},
|
||||
{
|
||||
"matches": 13604,
|
||||
"model": "llama-3.1-70b-instruct",
|
||||
"rank": 8,
|
||||
"rating": 1087.3529845908,
|
||||
"wins": 7058.5
|
||||
},
|
||||
{
|
||||
"matches": 77509,
|
||||
"model": "gpt-4o-2024-05-13",
|
||||
"rank": 9,
|
||||
"rating": 1086.7242770205,
|
||||
"wins": 46951.0
|
||||
},
|
||||
{
|
||||
"matches": 9761,
|
||||
"model": "gpt-4o-2024-08-06",
|
||||
"rank": 10,
|
||||
"rating": 1085.601246973,
|
||||
"wins": 5323.0
|
||||
},
|
||||
{
|
||||
"matches": 19370,
|
||||
"model": "gpt-4o-mini-2024-07-18",
|
||||
"rank": 11,
|
||||
"rating": 1079.187026245,
|
||||
"wins": 11082.0
|
||||
},
|
||||
{
|
||||
"matches": 47703,
|
||||
"model": "claude-3-5-sonnet-20240620",
|
||||
"rank": 12,
|
||||
"rating": 1079.0096726066,
|
||||
"wins": 27107.5
|
||||
},
|
||||
{
|
||||
"matches": 155944,
|
||||
"model": "claude-3-opus-20240229",
|
||||
"rank": 13,
|
||||
"rating": 1074.9078256001,
|
||||
"wins": 91432.5
|
||||
},
|
||||
{
|
||||
"matches": 11476,
|
||||
"model": "athene-70b-0725",
|
||||
"rank": 14,
|
||||
"rating": 1073.7607622933,
|
||||
"wins": 6098.0
|
||||
},
|
||||
{
|
||||
"matches": 18790,
|
||||
"model": "gemini-pro-dev-api",
|
||||
"rank": 15,
|
||||
"rating": 1072.3072263884,
|
||||
"wins": 8728.5
|
||||
},
|
||||
{
|
||||
"matches": 12774,
|
||||
"model": "claude-2.0",
|
||||
"rank": 16,
|
||||
"rating": 1069.530762176,
|
||||
"wins": 6814.5
|
||||
},
|
||||
{
|
||||
"matches": 10239,
|
||||
"model": "glm-4-0520",
|
||||
"rank": 17,
|
||||
"rating": 1065.2967580553,
|
||||
"wins": 5154.5
|
||||
},
|
||||
{
|
||||
"matches": 20669,
|
||||
"model": "nemotron-4-340b-instruct",
|
||||
"rank": 18,
|
||||
"rating": 1063.9797181947,
|
||||
"wins": 10567.5
|
||||
},
|
||||
{
|
||||
"matches": 51739,
|
||||
"model": "yi-large-preview",
|
||||
"rank": 19,
|
||||
"rating": 1060.8561460851,
|
||||
"wins": 28793.0
|
||||
},
|
||||
{
|
||||
"matches": 39631,
|
||||
"model": "llama-2-70b-chat",
|
||||
"rank": 20,
|
||||
"rating": 1060.6350445555,
|
||||
"wins": 18272.0
|
||||
}
|
||||
],
|
||||
"rank_comparison": {
|
||||
"claim_boundary": "This is a same-snapshot reconstruction of the official method, not a scrape of the mutable live leaderboard. Scores need not match the live service.",
|
||||
"common_models": 129,
|
||||
"comparison_target": "Bradley-Terry MLE reconstruction used by Chatbot Arena",
|
||||
"kendall_rank_correlation": 0.606347,
|
||||
"official_method_top_20": [
|
||||
"chatgpt-4o-latest",
|
||||
"gemini-1.5-pro-exp-0801",
|
||||
"gpt-4o-2024-05-13",
|
||||
"gpt-4o-mini-2024-07-18",
|
||||
"claude-3-5-sonnet-20240620",
|
||||
"gemini-advanced-0514",
|
||||
"llama-3.1-405b-instruct",
|
||||
"gpt-4o-2024-08-06",
|
||||
"gemini-1.5-pro-api-0514",
|
||||
"gemini-1.5-pro-api-0409-preview",
|
||||
"gpt-4-turbo-2024-04-09",
|
||||
"gpt-4-1106-preview",
|
||||
"mistral-large-2407",
|
||||
"athene-70b-0725",
|
||||
"claude-3-opus-20240229",
|
||||
"llama-3.1-70b-instruct",
|
||||
"gpt-4-0125-preview",
|
||||
"yi-large-preview",
|
||||
"reka-core-20240722",
|
||||
"gemini-1.5-flash-api-0514"
|
||||
],
|
||||
"online_top_20": [
|
||||
"gemini-1.5-pro-api-0409-preview",
|
||||
"gemini-1.5-pro-exp-0801",
|
||||
"chatgpt-4o-latest",
|
||||
"gpt-3.5-turbo-0314",
|
||||
"bard-jan-24-gemini-pro",
|
||||
"claude-1",
|
||||
"gemini-advanced-0514",
|
||||
"llama-3.1-70b-instruct",
|
||||
"gpt-4o-2024-05-13",
|
||||
"gpt-4o-2024-08-06",
|
||||
"gpt-4o-mini-2024-07-18",
|
||||
"claude-3-5-sonnet-20240620",
|
||||
"claude-3-opus-20240229",
|
||||
"athene-70b-0725",
|
||||
"gemini-pro-dev-api",
|
||||
"claude-2.0",
|
||||
"glm-4-0520",
|
||||
"nemotron-4-340b-instruct",
|
||||
"yi-large-preview",
|
||||
"llama-2-70b-chat"
|
||||
],
|
||||
"spearman_rank_correlation": 0.786717,
|
||||
"top_20_overlap": 12
|
||||
}
|
||||
},
|
||||
"schema_version": 1,
|
||||
"status": "passed",
|
||||
"timing_seconds": {
|
||||
"bradley_terry": 71.472,
|
||||
"online_and_history": 8.301,
|
||||
"total": 97.248
|
||||
}
|
||||
}
|
||||
+908
@@ -0,0 +1,908 @@
|
||||
{
|
||||
"empirical": {
|
||||
"athene-70b-0725": {
|
||||
"athene-70b-0725": 0.5,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": 0.41589648798521256,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.49077181208053694,
|
||||
"claude-3-opus-20240229": 0.5027472527472527,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": 0.43097014925373134,
|
||||
"gemini-advanced-0514": 0.48249027237354086,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.4431239388794567,
|
||||
"gpt-4o-2024-08-06": 0.505249343832021,
|
||||
"gpt-4o-mini-2024-07-18": 0.4753787878787879,
|
||||
"llama-2-70b-chat": NaN,
|
||||
"llama-3.1-70b-instruct": 0.5317460317460317,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": 0.5294117647058824
|
||||
},
|
||||
"bard-jan-24-gemini-pro": {
|
||||
"athene-70b-0725": NaN,
|
||||
"bard-jan-24-gemini-pro": 0.5,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": 0.572289156626506,
|
||||
"claude-2.0": 0.6696428571428571,
|
||||
"claude-3-5-sonnet-20240620": NaN,
|
||||
"claude-3-opus-20240229": 0.3,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": NaN,
|
||||
"gemini-advanced-0514": NaN,
|
||||
"gemini-pro-dev-api": 0.6332794830371568,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": NaN,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": NaN,
|
||||
"llama-2-70b-chat": 0.6492537313432836,
|
||||
"llama-3.1-70b-instruct": NaN,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": NaN
|
||||
},
|
||||
"chatgpt-4o-latest": {
|
||||
"athene-70b-0725": 0.5841035120147874,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": 0.5,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.5534351145038168,
|
||||
"claude-3-opus-20240229": 0.5915750915750916,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": 0.5329736211031175,
|
||||
"gemini-advanced-0514": NaN,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.5442092154420921,
|
||||
"gpt-4o-2024-08-06": 0.5164835164835165,
|
||||
"gpt-4o-mini-2024-07-18": 0.6010733452593918,
|
||||
"llama-2-70b-chat": NaN,
|
||||
"llama-3.1-70b-instruct": 0.5793838862559242,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": NaN
|
||||
},
|
||||
"claude-1": {
|
||||
"athene-70b-0725": NaN,
|
||||
"bard-jan-24-gemini-pro": 0.42771084337349397,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": 0.5,
|
||||
"claude-2.0": 0.5460750853242321,
|
||||
"claude-3-5-sonnet-20240620": NaN,
|
||||
"claude-3-opus-20240229": 0.34452296819787986,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": NaN,
|
||||
"gemini-advanced-0514": NaN,
|
||||
"gemini-pro-dev-api": 0.5342465753424658,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": 0.5548780487804879,
|
||||
"gpt-4o-2024-05-13": NaN,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": NaN,
|
||||
"llama-2-70b-chat": 0.604,
|
||||
"llama-3.1-70b-instruct": NaN,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": NaN
|
||||
},
|
||||
"claude-2.0": {
|
||||
"athene-70b-0725": NaN,
|
||||
"bard-jan-24-gemini-pro": 0.33035714285714285,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": 0.4539249146757679,
|
||||
"claude-2.0": 0.5,
|
||||
"claude-3-5-sonnet-20240620": NaN,
|
||||
"claude-3-opus-20240229": NaN,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": NaN,
|
||||
"gemini-advanced-0514": NaN,
|
||||
"gemini-pro-dev-api": 0.5569620253164557,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": NaN,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": NaN,
|
||||
"llama-2-70b-chat": 0.5866834170854272,
|
||||
"llama-3.1-70b-instruct": NaN,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": NaN
|
||||
},
|
||||
"claude-3-5-sonnet-20240620": {
|
||||
"athene-70b-0725": 0.5092281879194631,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": 0.44656488549618323,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.5,
|
||||
"claude-3-opus-20240229": 0.5447761194029851,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": 0.4537037037037037,
|
||||
"gemini-advanced-0514": 0.49312714776632305,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": 0.5800756620428752,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.47875730217737655,
|
||||
"gpt-4o-2024-08-06": 0.5189873417721519,
|
||||
"gpt-4o-mini-2024-07-18": 0.47484909456740443,
|
||||
"llama-2-70b-chat": NaN,
|
||||
"llama-3.1-70b-instruct": 0.538961038961039,
|
||||
"nemotron-4-340b-instruct": 0.589835728952772,
|
||||
"yi-large-preview": 0.5205761316872428
|
||||
},
|
||||
"claude-3-opus-20240229": {
|
||||
"athene-70b-0725": 0.49725274725274726,
|
||||
"bard-jan-24-gemini-pro": 0.7,
|
||||
"chatgpt-4o-latest": 0.4084249084249084,
|
||||
"claude-1": 0.6554770318021201,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.4552238805970149,
|
||||
"claude-3-opus-20240229": 0.5,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.4815616343490305,
|
||||
"gemini-1.5-pro-exp-0801": 0.4305157593123209,
|
||||
"gemini-advanced-0514": 0.46121231155778897,
|
||||
"gemini-pro-dev-api": 0.6489533011272142,
|
||||
"glm-4-0520": 0.5424311926605505,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.4481694338016177,
|
||||
"gpt-4o-2024-08-06": 0.4369627507163324,
|
||||
"gpt-4o-mini-2024-07-18": 0.4437386569872958,
|
||||
"llama-2-70b-chat": 0.6890951276102089,
|
||||
"llama-3.1-70b-instruct": 0.4813519813519814,
|
||||
"nemotron-4-340b-instruct": 0.5431309904153354,
|
||||
"yi-large-preview": 0.5238190552441954
|
||||
},
|
||||
"gemini-1.5-pro-api-0409-preview": {
|
||||
"athene-70b-0725": NaN,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": NaN,
|
||||
"claude-3-opus-20240229": 0.5184383656509696,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.5,
|
||||
"gemini-1.5-pro-exp-0801": NaN,
|
||||
"gemini-advanced-0514": NaN,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": NaN,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": NaN,
|
||||
"llama-2-70b-chat": 0.6875,
|
||||
"llama-3.1-70b-instruct": NaN,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": NaN
|
||||
},
|
||||
"gemini-1.5-pro-exp-0801": {
|
||||
"athene-70b-0725": 0.5690298507462687,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": 0.4670263788968825,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.5462962962962963,
|
||||
"claude-3-opus-20240229": 0.5694842406876791,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": 0.5,
|
||||
"gemini-advanced-0514": 0.5263157894736842,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.5229445506692161,
|
||||
"gpt-4o-2024-08-06": 0.5241071428571429,
|
||||
"gpt-4o-mini-2024-07-18": 0.529559748427673,
|
||||
"llama-2-70b-chat": NaN,
|
||||
"llama-3.1-70b-instruct": 0.5773092369477911,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": 0.5862068965517241
|
||||
},
|
||||
"gemini-advanced-0514": {
|
||||
"athene-70b-0725": 0.5175097276264592,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.506872852233677,
|
||||
"claude-3-opus-20240229": 0.538787688442211,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": 0.47368421052631576,
|
||||
"gemini-advanced-0514": 0.5,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": 0.5773381294964028,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.4901780233271946,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": 0.4604072398190045,
|
||||
"llama-2-70b-chat": 0.6145833333333334,
|
||||
"llama-3.1-70b-instruct": 0.5423076923076923,
|
||||
"nemotron-4-340b-instruct": 0.5807407407407408,
|
||||
"yi-large-preview": 0.5602636534839924
|
||||
},
|
||||
"gemini-pro-dev-api": {
|
||||
"athene-70b-0725": NaN,
|
||||
"bard-jan-24-gemini-pro": 0.3667205169628433,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": 0.4657534246575342,
|
||||
"claude-2.0": 0.4430379746835443,
|
||||
"claude-3-5-sonnet-20240620": NaN,
|
||||
"claude-3-opus-20240229": 0.35104669887278583,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": NaN,
|
||||
"gemini-advanced-0514": NaN,
|
||||
"gemini-pro-dev-api": 0.5,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": NaN,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": NaN,
|
||||
"llama-2-70b-chat": 0.5535714285714286,
|
||||
"llama-3.1-70b-instruct": NaN,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": NaN
|
||||
},
|
||||
"glm-4-0520": {
|
||||
"athene-70b-0725": NaN,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.41992433795712486,
|
||||
"claude-3-opus-20240229": 0.45756880733944955,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": NaN,
|
||||
"gemini-advanced-0514": 0.4226618705035971,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": 0.5,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.3613861386138614,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": NaN,
|
||||
"llama-2-70b-chat": NaN,
|
||||
"llama-3.1-70b-instruct": NaN,
|
||||
"nemotron-4-340b-instruct": 0.5087025316455697,
|
||||
"yi-large-preview": 0.448943661971831
|
||||
},
|
||||
"gpt-3.5-turbo-0314": {
|
||||
"athene-70b-0725": NaN,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": 0.4451219512195122,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": NaN,
|
||||
"claude-3-opus-20240229": NaN,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": NaN,
|
||||
"gemini-advanced-0514": NaN,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": 0.5,
|
||||
"gpt-4o-2024-05-13": NaN,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": NaN,
|
||||
"llama-2-70b-chat": NaN,
|
||||
"llama-3.1-70b-instruct": NaN,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": NaN
|
||||
},
|
||||
"gpt-4o-2024-05-13": {
|
||||
"athene-70b-0725": 0.5568760611205433,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": 0.45579078455790784,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.5212426978226234,
|
||||
"claude-3-opus-20240229": 0.5518305661983823,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": 0.4770554493307839,
|
||||
"gemini-advanced-0514": 0.5098219766728054,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": 0.6386138613861386,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.5,
|
||||
"gpt-4o-2024-08-06": 0.544973544973545,
|
||||
"gpt-4o-mini-2024-07-18": 0.5163472378804961,
|
||||
"llama-2-70b-chat": 0.5607476635514018,
|
||||
"llama-3.1-70b-instruct": 0.5650224215246636,
|
||||
"nemotron-4-340b-instruct": 0.5968559837728195,
|
||||
"yi-large-preview": 0.5756445047489823
|
||||
},
|
||||
"gpt-4o-2024-08-06": {
|
||||
"athene-70b-0725": 0.494750656167979,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": 0.4835164835164835,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.4810126582278481,
|
||||
"claude-3-opus-20240229": 0.5630372492836676,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": 0.4758928571428571,
|
||||
"gemini-advanced-0514": NaN,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.455026455026455,
|
||||
"gpt-4o-2024-08-06": 0.5,
|
||||
"gpt-4o-mini-2024-07-18": 0.5052910052910053,
|
||||
"llama-2-70b-chat": NaN,
|
||||
"llama-3.1-70b-instruct": 0.5273037542662116,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": NaN
|
||||
},
|
||||
"gpt-4o-mini-2024-07-18": {
|
||||
"athene-70b-0725": 0.5246212121212122,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": 0.39892665474060823,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.5251509054325956,
|
||||
"claude-3-opus-20240229": 0.5562613430127041,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": 0.47044025157232705,
|
||||
"gemini-advanced-0514": 0.5395927601809954,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.4836527621195039,
|
||||
"gpt-4o-2024-08-06": 0.4947089947089947,
|
||||
"gpt-4o-mini-2024-07-18": 0.5,
|
||||
"llama-2-70b-chat": NaN,
|
||||
"llama-3.1-70b-instruct": 0.5584756898817346,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": 0.553125
|
||||
},
|
||||
"llama-2-70b-chat": {
|
||||
"athene-70b-0725": NaN,
|
||||
"bard-jan-24-gemini-pro": 0.35074626865671643,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": 0.396,
|
||||
"claude-2.0": 0.41331658291457285,
|
||||
"claude-3-5-sonnet-20240620": NaN,
|
||||
"claude-3-opus-20240229": 0.3109048723897912,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3125,
|
||||
"gemini-1.5-pro-exp-0801": NaN,
|
||||
"gemini-advanced-0514": 0.3854166666666667,
|
||||
"gemini-pro-dev-api": 0.44642857142857145,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.4392523364485981,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": NaN,
|
||||
"llama-2-70b-chat": 0.5,
|
||||
"llama-3.1-70b-instruct": NaN,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": 0.42142857142857143
|
||||
},
|
||||
"llama-3.1-70b-instruct": {
|
||||
"athene-70b-0725": 0.46825396825396826,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": 0.4206161137440758,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.461038961038961,
|
||||
"claude-3-opus-20240229": 0.5186480186480187,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": 0.4226907630522088,
|
||||
"gemini-advanced-0514": 0.4576923076923077,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": NaN,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.4349775784753363,
|
||||
"gpt-4o-2024-08-06": 0.4726962457337884,
|
||||
"gpt-4o-mini-2024-07-18": 0.44152431011826543,
|
||||
"llama-2-70b-chat": NaN,
|
||||
"llama-3.1-70b-instruct": 0.5,
|
||||
"nemotron-4-340b-instruct": NaN,
|
||||
"yi-large-preview": 0.58
|
||||
},
|
||||
"nemotron-4-340b-instruct": {
|
||||
"athene-70b-0725": NaN,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.4101642710472279,
|
||||
"claude-3-opus-20240229": 0.45686900958466453,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": NaN,
|
||||
"gemini-advanced-0514": 0.4192592592592593,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": 0.4912974683544304,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.40314401622718055,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": NaN,
|
||||
"llama-2-70b-chat": NaN,
|
||||
"llama-3.1-70b-instruct": NaN,
|
||||
"nemotron-4-340b-instruct": 0.5,
|
||||
"yi-large-preview": 0.44568690095846647
|
||||
},
|
||||
"yi-large-preview": {
|
||||
"athene-70b-0725": 0.47058823529411764,
|
||||
"bard-jan-24-gemini-pro": NaN,
|
||||
"chatgpt-4o-latest": NaN,
|
||||
"claude-1": NaN,
|
||||
"claude-2.0": NaN,
|
||||
"claude-3-5-sonnet-20240620": 0.4794238683127572,
|
||||
"claude-3-opus-20240229": 0.47618094475580464,
|
||||
"gemini-1.5-pro-api-0409-preview": NaN,
|
||||
"gemini-1.5-pro-exp-0801": 0.41379310344827586,
|
||||
"gemini-advanced-0514": 0.4397363465160075,
|
||||
"gemini-pro-dev-api": NaN,
|
||||
"glm-4-0520": 0.551056338028169,
|
||||
"gpt-3.5-turbo-0314": NaN,
|
||||
"gpt-4o-2024-05-13": 0.4243554952510176,
|
||||
"gpt-4o-2024-08-06": NaN,
|
||||
"gpt-4o-mini-2024-07-18": 0.446875,
|
||||
"llama-2-70b-chat": 0.5785714285714286,
|
||||
"llama-3.1-70b-instruct": 0.42,
|
||||
"nemotron-4-340b-instruct": 0.5543130990415336,
|
||||
"yi-large-preview": 0.5
|
||||
}
|
||||
},
|
||||
"models": [
|
||||
"gemini-1.5-pro-api-0409-preview",
|
||||
"gemini-1.5-pro-exp-0801",
|
||||
"chatgpt-4o-latest",
|
||||
"gpt-3.5-turbo-0314",
|
||||
"bard-jan-24-gemini-pro",
|
||||
"claude-1",
|
||||
"gemini-advanced-0514",
|
||||
"llama-3.1-70b-instruct",
|
||||
"gpt-4o-2024-05-13",
|
||||
"gpt-4o-2024-08-06",
|
||||
"gpt-4o-mini-2024-07-18",
|
||||
"claude-3-5-sonnet-20240620",
|
||||
"claude-3-opus-20240229",
|
||||
"athene-70b-0725",
|
||||
"gemini-pro-dev-api",
|
||||
"claude-2.0",
|
||||
"glm-4-0520",
|
||||
"nemotron-4-340b-instruct",
|
||||
"yi-large-preview",
|
||||
"llama-2-70b-chat"
|
||||
],
|
||||
"online_elo_predicted": {
|
||||
"athene-70b-0725": {
|
||||
"athene-70b-0725": 0.5,
|
||||
"bard-jan-24-gemini-pro": 0.4706024381703526,
|
||||
"chatgpt-4o-latest": 0.4244076072370654,
|
||||
"claude-1": 0.4727267496028405,
|
||||
"claude-2.0": 0.5060871587475949,
|
||||
"claude-3-5-sonnet-20240620": 0.4924467854859755,
|
||||
"claude-3-opus-20240229": 0.4983492492034077,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3786225197416955,
|
||||
"gemini-1.5-pro-exp-0801": 0.4227290744565508,
|
||||
"gemini-advanced-0514": 0.48017931822260296,
|
||||
"gemini-pro-dev-api": 0.5020917941129024,
|
||||
"glm-4-0520": 0.5121782721655728,
|
||||
"gpt-3.5-turbo-0314": 0.45662732701186726,
|
||||
"gpt-4o-2024-05-13": 0.4813526553622729,
|
||||
"gpt-4o-2024-08-06": 0.482966766553394,
|
||||
"gpt-4o-mini-2024-07-18": 0.492191613318456,
|
||||
"llama-2-70b-chat": 0.5188804447839834,
|
||||
"llama-3.1-70b-instruct": 0.4804491928529648,
|
||||
"nemotron-4-340b-instruct": 0.5140723365206418,
|
||||
"yi-large-preview": 0.5185627002158419
|
||||
},
|
||||
"bard-jan-24-gemini-pro": {
|
||||
"athene-70b-0725": 0.5293975618296474,
|
||||
"bard-jan-24-gemini-pro": 0.5,
|
||||
"chatgpt-4o-latest": 0.45339086380344346,
|
||||
"claude-1": 0.5021311461637057,
|
||||
"claude-2.0": 0.5354593391190725,
|
||||
"claude-3-5-sonnet-20240620": 0.5218637663710245,
|
||||
"claude-3-opus-20240229": 0.5277521980731233,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.4066882610443156,
|
||||
"gemini-1.5-pro-exp-0801": 0.45168765548015544,
|
||||
"gemini-advanced-0514": 0.5095992532128728,
|
||||
"gemini-pro-dev-api": 0.5314816122659721,
|
||||
"glm-4-0520": 0.54151638062692,
|
||||
"gpt-3.5-turbo-0314": 0.48595324754769137,
|
||||
"gpt-4o-2024-05-13": 0.5107738414883239,
|
||||
"gpt-4o-2024-08-06": 0.5123891431197954,
|
||||
"gpt-4o-mini-2024-07-18": 0.5216090163334262,
|
||||
"llama-2-70b-chat": 0.5481710593389454,
|
||||
"llama-3.1-70b-instruct": 0.5098694443797749,
|
||||
"nemotron-4-340b-instruct": 0.5433980845222826,
|
||||
"yi-large-preview": 0.5478558027577041
|
||||
},
|
||||
"chatgpt-4o-latest": {
|
||||
"athene-70b-0725": 0.5755923927629346,
|
||||
"bard-jan-24-gemini-pro": 0.5466091361965566,
|
||||
"chatgpt-4o-latest": 0.5,
|
||||
"claude-1": 0.5487209243907352,
|
||||
"claude-2.0": 0.5815294906471586,
|
||||
"claude-3-5-sonnet-20240620": 0.5681949260650095,
|
||||
"claude-3-opus-20240229": 0.5739785673924995,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.45247054063006786,
|
||||
"gemini-1.5-pro-exp-0801": 0.49828131112534013,
|
||||
"gemini-advanced-0514": 0.5561079756808931,
|
||||
"gemini-pro-dev-api": 0.5776350830826076,
|
||||
"glm-4-0520": 0.587448649362509,
|
||||
"gpt-3.5-turbo-0314": 0.5326478829339771,
|
||||
"gpt-4o-2024-05-13": 0.5572679471226406,
|
||||
"gpt-4o-2024-08-06": 0.5588623197754942,
|
||||
"gpt-4o-mini-2024-07-18": 0.5679444241258642,
|
||||
"llama-2-70b-chat": 0.5939365657179976,
|
||||
"llama-3.1-70b-instruct": 0.5563748494378427,
|
||||
"nemotron-4-340b-instruct": 0.5892848182429368,
|
||||
"yi-large-preview": 0.5936295693344523
|
||||
},
|
||||
"claude-1": {
|
||||
"athene-70b-0725": 0.5272732503971594,
|
||||
"bard-jan-24-gemini-pro": 0.4978688538362943,
|
||||
"chatgpt-4o-latest": 0.45127907560926495,
|
||||
"claude-1": 0.5,
|
||||
"claude-2.0": 0.5333382703189705,
|
||||
"claude-3-5-sonnet-20240620": 0.5197362986493318,
|
||||
"claude-3-opus-20240229": 0.5256271146687194,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.4046329739918908,
|
||||
"gemini-1.5-pro-exp-0801": 0.44957727554595445,
|
||||
"gemini-advanced-0514": 0.5074687182117393,
|
||||
"gemini-pro-dev-api": 0.5293583449328568,
|
||||
"glm-4-0520": 0.539399178222487,
|
||||
"gpt-3.5-turbo-0314": 0.483824038337802,
|
||||
"gpt-4o-2024-05-13": 0.5086434891644779,
|
||||
"gpt-4o-2024-08-06": 0.5102590804411641,
|
||||
"gpt-4o-mini-2024-07-18": 0.519481458808123,
|
||||
"llama-2-70b-chat": 0.5460588266923004,
|
||||
"llama-3.1-70b-instruct": 0.5077389493164274,
|
||||
"nemotron-4-340b-instruct": 0.541282210739082,
|
||||
"yi-large-preview": 0.5457433176189294
|
||||
},
|
||||
"claude-2.0": {
|
||||
"athene-70b-0725": 0.4939128412524052,
|
||||
"bard-jan-24-gemini-pro": 0.46454066088092755,
|
||||
"chatgpt-4o-latest": 0.41847050935284147,
|
||||
"claude-1": 0.46666172968102954,
|
||||
"claude-2.0": 0.5,
|
||||
"claude-3-5-sonnet-20240620": 0.4863621348844711,
|
||||
"claude-3-opus-20240229": 0.4922624014572012,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3729109568972296,
|
||||
"gemini-1.5-pro-exp-0801": 0.41679845455453196,
|
||||
"gemini-advanced-0514": 0.474104656737174,
|
||||
"gemini-pro-dev-api": 0.4960044318617072,
|
||||
"glm-4-0520": 0.5060929201168735,
|
||||
"gpt-3.5-turbo-0314": 0.4505923459773481,
|
||||
"gpt-4o-2024-05-13": 0.4752767219072891,
|
||||
"gpt-4o-2024-08-06": 0.4768891926892078,
|
||||
"gpt-4o-mini-2024-07-18": 0.4861070959392006,
|
||||
"llama-2-70b-chat": 0.5127991699819673,
|
||||
"llama-3.1-70b-instruct": 0.47437423287938824,
|
||||
"nemotron-4-340b-instruct": 0.5079879147696204,
|
||||
"yi-large-preview": 0.512481182668407
|
||||
},
|
||||
"claude-3-5-sonnet-20240620": {
|
||||
"athene-70b-0725": 0.5075532145140246,
|
||||
"bard-jan-24-gemini-pro": 0.4781362336289755,
|
||||
"chatgpt-4o-latest": 0.4318050739349906,
|
||||
"claude-1": 0.4802637013506682,
|
||||
"claude-2.0": 0.5136378651155289,
|
||||
"claude-3-5-sonnet-20240620": 0.5,
|
||||
"claude-3-opus-20240229": 0.505902758110997,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.38575678604471353,
|
||||
"gemini-1.5-pro-exp-0801": 0.4301191468425039,
|
||||
"gemini-advanced-0514": 0.4877251820914822,
|
||||
"gemini-pro-dev-api": 0.5096443991097894,
|
||||
"glm-4-0520": 0.519724229338601,
|
||||
"gpt-3.5-turbo-0314": 0.46413354167370846,
|
||||
"gpt-4o-2024-05-13": 0.48889961603567494,
|
||||
"gpt-4o-2024-08-06": 0.4905150999228697,
|
||||
"gpt-4o-mini-2024-07-18": 0.49974476761967085,
|
||||
"llama-2-70b-chat": 0.526418589284047,
|
||||
"llama-3.1-70b-instruct": 0.487995316371302,
|
||||
"nemotron-4-340b-instruct": 0.5216163605038188,
|
||||
"yi-large-preview": 0.5261012762929619
|
||||
},
|
||||
"claude-3-opus-20240229": {
|
||||
"athene-70b-0725": 0.5016507507965924,
|
||||
"bard-jan-24-gemini-pro": 0.47224780192687665,
|
||||
"chatgpt-4o-latest": 0.4260214326075004,
|
||||
"claude-1": 0.47437288533128064,
|
||||
"claude-2.0": 0.5077375985427988,
|
||||
"claude-3-5-sonnet-20240620": 0.49409724188900295,
|
||||
"claude-3-opus-20240229": 0.5,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3801772378799626,
|
||||
"gemini-1.5-pro-exp-0801": 0.4243412226190692,
|
||||
"gemini-advanced-0514": 0.4818276906995902,
|
||||
"gemini-pro-dev-api": 0.5037424932177174,
|
||||
"glm-4-0520": 0.513827911016007,
|
||||
"gpt-3.5-turbo-0314": 0.4582661256484433,
|
||||
"gpt-4o-2024-05-13": 0.4830013131364567,
|
||||
"gpt-4o-2024-08-06": 0.48461578707994823,
|
||||
"gpt-4o-mini-2024-07-18": 0.4938420466183484,
|
||||
"llama-2-70b-chat": 0.520528636323993,
|
||||
"llama-3.1-70b-instruct": 0.48209763256058513,
|
||||
"nemotron-4-340b-instruct": 0.5157216264686881,
|
||||
"yi-large-preview": 0.5202109737620978
|
||||
},
|
||||
"gemini-1.5-pro-api-0409-preview": {
|
||||
"athene-70b-0725": 0.6213774802583045,
|
||||
"bard-jan-24-gemini-pro": 0.5933117389556843,
|
||||
"chatgpt-4o-latest": 0.5475294593699321,
|
||||
"claude-1": 0.5953670260081093,
|
||||
"claude-2.0": 0.6270890431027705,
|
||||
"claude-3-5-sonnet-20240620": 0.6142432139552865,
|
||||
"claude-3-opus-20240229": 0.6198227621200374,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.5,
|
||||
"gemini-1.5-pro-exp-0801": 0.5458257442135583,
|
||||
"gemini-advanced-0514": 0.6025435895577902,
|
||||
"gemini-pro-dev-api": 0.6233440078256695,
|
||||
"glm-4-0520": 0.6327707226950309,
|
||||
"gpt-3.5-turbo-0314": 0.579682755828864,
|
||||
"gpt-4o-2024-05-13": 0.6036686971453942,
|
||||
"gpt-4o-2024-08-06": 0.6052143488362124,
|
||||
"gpt-4o-mini-2024-07-18": 0.6140012780307695,
|
||||
"llama-2-70b-chat": 0.6389839091289585,
|
||||
"llama-3.1-70b-instruct": 0.6028024856995498,
|
||||
"nemotron-4-340b-instruct": 0.6345306684098833,
|
||||
"yi-large-preview": 0.6386902518991052
|
||||
},
|
||||
"gemini-1.5-pro-exp-0801": {
|
||||
"athene-70b-0725": 0.5772709255434492,
|
||||
"bard-jan-24-gemini-pro": 0.5483123445198447,
|
||||
"chatgpt-4o-latest": 0.5017186888746599,
|
||||
"claude-1": 0.5504227244540455,
|
||||
"claude-2.0": 0.5832015454454681,
|
||||
"claude-3-5-sonnet-20240620": 0.569880853157496,
|
||||
"claude-3-opus-20240229": 0.5756587773809309,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.4541742557864416,
|
||||
"gemini-1.5-pro-exp-0801": 0.5,
|
||||
"gemini-advanced-0514": 0.557804367756876,
|
||||
"gemini-pro-dev-api": 0.5793114416789945,
|
||||
"glm-4-0520": 0.5891137641041534,
|
||||
"gpt-3.5-turbo-0314": 0.5343588600926158,
|
||||
"gpt-4o-2024-05-13": 0.5589634219132797,
|
||||
"gpt-4o-2024-08-06": 0.5605565036097535,
|
||||
"gpt-4o-mini-2024-07-18": 0.5696305884618997,
|
||||
"llama-2-70b-chat": 0.5955935211645167,
|
||||
"llama-3.1-70b-instruct": 0.5580710321113055,
|
||||
"nemotron-4-340b-instruct": 0.5909476824028691,
|
||||
"yi-large-preview": 0.5952869238841539
|
||||
},
|
||||
"gemini-advanced-0514": {
|
||||
"athene-70b-0725": 0.519820681777397,
|
||||
"bard-jan-24-gemini-pro": 0.49040074678712725,
|
||||
"chatgpt-4o-latest": 0.4438920243191068,
|
||||
"claude-1": 0.4925312817882607,
|
||||
"claude-2.0": 0.525895343262826,
|
||||
"claude-3-5-sonnet-20240620": 0.5122748179085177,
|
||||
"claude-3-opus-20240229": 0.5181723093004097,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3974564104422098,
|
||||
"gemini-1.5-pro-exp-0801": 0.44219563224312397,
|
||||
"gemini-advanced-0514": 0.5,
|
||||
"gemini-pro-dev-api": 0.5219088424590324,
|
||||
"glm-4-0520": 0.5319680879028288,
|
||||
"gpt-3.5-turbo-0314": 0.4763667410118573,
|
||||
"gpt-4o-2024-05-13": 0.5011750743841354,
|
||||
"gpt-4o-2024-08-06": 0.5027912177061772,
|
||||
"gpt-4o-mini-2024-07-18": 0.5120197361562043,
|
||||
"llama-2-70b-chat": 0.5386432816976552,
|
||||
"llama-3.1-70b-instruct": 0.5002702935966019,
|
||||
"nemotron-4-340b-instruct": 0.5338552462294891,
|
||||
"yi-large-preview": 0.5383269761251527
|
||||
},
|
||||
"gemini-pro-dev-api": {
|
||||
"athene-70b-0725": 0.4979082058870976,
|
||||
"bard-jan-24-gemini-pro": 0.4685183877340279,
|
||||
"chatgpt-4o-latest": 0.4223649169173924,
|
||||
"claude-1": 0.47064165506714317,
|
||||
"claude-2.0": 0.5039955681382927,
|
||||
"claude-3-5-sonnet-20240620": 0.4903556008902106,
|
||||
"claude-3-opus-20240229": 0.4962575067822826,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.37665599217433055,
|
||||
"gemini-1.5-pro-exp-0801": 0.4206885583210056,
|
||||
"gemini-advanced-0514": 0.4780911575409677,
|
||||
"gemini-pro-dev-api": 0.5,
|
||||
"glm-4-0520": 0.5100875059468505,
|
||||
"gpt-3.5-turbo-0314": 0.45455202627804153,
|
||||
"gpt-4o-2024-05-13": 0.47926409658161856,
|
||||
"gpt-4o-2024-08-06": 0.4808776977523424,
|
||||
"gpt-4o-mini-2024-07-18": 0.49010046598318574,
|
||||
"llama-2-70b-chat": 0.5167913032942285,
|
||||
"llama-3.1-70b-instruct": 0.4783609385670775,
|
||||
"nemotron-4-340b-instruct": 0.5119819532314825,
|
||||
"yi-large-preview": 0.516473464718453
|
||||
},
|
||||
"glm-4-0520": {
|
||||
"athene-70b-0725": 0.4878217278344273,
|
||||
"bard-jan-24-gemini-pro": 0.4584836193730801,
|
||||
"chatgpt-4o-latest": 0.41255135063749104,
|
||||
"claude-1": 0.4606008217775131,
|
||||
"claude-2.0": 0.49390707988312654,
|
||||
"claude-3-5-sonnet-20240620": 0.48027577066139915,
|
||||
"claude-3-opus-20240229": 0.48617208898399294,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3672292773049691,
|
||||
"gemini-1.5-pro-exp-0801": 0.4108862358958465,
|
||||
"gemini-advanced-0514": 0.4680319120971712,
|
||||
"gemini-pro-dev-api": 0.4899124940531495,
|
||||
"glm-4-0520": 0.5,
|
||||
"gpt-3.5-turbo-0314": 0.444566176363753,
|
||||
"gpt-4o-2024-05-13": 0.4692023588423809,
|
||||
"gpt-4o-2024-08-06": 0.47081271228911237,
|
||||
"gpt-4o-mini-2024-07-18": 0.4800209406003484,
|
||||
"llama-2-70b-chat": 0.5067083424471972,
|
||||
"llama-3.1-70b-instruct": 0.4683011100825741,
|
||||
"nemotron-4-340b-instruct": 0.5018953636400675,
|
||||
"yi-large-preview": 0.5063902063717687
|
||||
},
|
||||
"gpt-3.5-turbo-0314": {
|
||||
"athene-70b-0725": 0.5433726729881329,
|
||||
"bard-jan-24-gemini-pro": 0.5140467524523086,
|
||||
"chatgpt-4o-latest": 0.4673521170660229,
|
||||
"claude-1": 0.5161759616621979,
|
||||
"claude-2.0": 0.5494076540226519,
|
||||
"claude-3-5-sonnet-20240620": 0.5358664583262915,
|
||||
"claude-3-opus-20240229": 0.5417338743515567,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.4203172441711361,
|
||||
"gemini-1.5-pro-exp-0801": 0.4656411399073841,
|
||||
"gemini-advanced-0514": 0.5236332589881427,
|
||||
"gemini-pro-dev-api": 0.5454479737219585,
|
||||
"glm-4-0520": 0.5554338236362469,
|
||||
"gpt-3.5-turbo-0314": 0.5,
|
||||
"gpt-4o-2024-05-13": 0.5248055778856127,
|
||||
"gpt-4o-2024-08-06": 0.5264175061108259,
|
||||
"gpt-4o-mini-2024-07-18": 0.5356125299744643,
|
||||
"llama-2-70b-chat": 0.5620498683755348,
|
||||
"llama-3.1-70b-instruct": 0.5239029418245605,
|
||||
"nemotron-4-340b-instruct": 0.5573051037168208,
|
||||
"yi-large-preview": 0.5617365533684751
|
||||
},
|
||||
"gpt-4o-2024-05-13": {
|
||||
"athene-70b-0725": 0.5186473446377271,
|
||||
"bard-jan-24-gemini-pro": 0.48922615851167606,
|
||||
"chatgpt-4o-latest": 0.4427320528773593,
|
||||
"claude-1": 0.49135651083552223,
|
||||
"claude-2.0": 0.5247232780927109,
|
||||
"claude-3-5-sonnet-20240620": 0.511100383964325,
|
||||
"claude-3-opus-20240229": 0.5169986868635432,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3963313028546058,
|
||||
"gemini-1.5-pro-exp-0801": 0.44103657808672025,
|
||||
"gemini-advanced-0514": 0.4988249256158645,
|
||||
"gemini-pro-dev-api": 0.5207359034183815,
|
||||
"glm-4-0520": 0.5307976411576192,
|
||||
"gpt-3.5-turbo-0314": 0.4751944221143874,
|
||||
"gpt-4o-2024-05-13": 0.5,
|
||||
"gpt-4o-2024-08-06": 0.501616164525399,
|
||||
"gpt-4o-mini-2024-07-18": 0.5108452744903431,
|
||||
"llama-2-70b-chat": 0.5374750140847713,
|
||||
"llama-3.1-70b-instruct": 0.4990952180629769,
|
||||
"nemotron-4-340b-instruct": 0.5326853730599583,
|
||||
"yi-large-preview": 0.537158595794847
|
||||
},
|
||||
"gpt-4o-2024-08-06": {
|
||||
"athene-70b-0725": 0.517033233446606,
|
||||
"bard-jan-24-gemini-pro": 0.48761085688020456,
|
||||
"chatgpt-4o-latest": 0.44113768022450583,
|
||||
"claude-1": 0.4897409195588359,
|
||||
"claude-2.0": 0.5231108073107922,
|
||||
"claude-3-5-sonnet-20240620": 0.5094849000771304,
|
||||
"claude-3-opus-20240229": 0.5153842129200518,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.39478565116378755,
|
||||
"gemini-1.5-pro-exp-0801": 0.4394434963902465,
|
||||
"gemini-advanced-0514": 0.4972087822938228,
|
||||
"gemini-pro-dev-api": 0.5191223022476575,
|
||||
"glm-4-0520": 0.5291872877108876,
|
||||
"gpt-3.5-turbo-0314": 0.473582493889174,
|
||||
"gpt-4o-2024-05-13": 0.4983838354746009,
|
||||
"gpt-4o-2024-08-06": 0.5,
|
||||
"gpt-4o-mini-2024-07-18": 0.5092297570723646,
|
||||
"llama-2-70b-chat": 0.5358675389424615,
|
||||
"llama-3.1-70b-instruct": 0.49747906828277433,
|
||||
"nemotron-4-340b-instruct": 0.531075774838379,
|
||||
"yi-large-preview": 0.5355509712390535
|
||||
},
|
||||
"gpt-4o-mini-2024-07-18": {
|
||||
"athene-70b-0725": 0.507808386681544,
|
||||
"bard-jan-24-gemini-pro": 0.47839098366657384,
|
||||
"chatgpt-4o-latest": 0.43205557587413584,
|
||||
"claude-1": 0.48051854119187704,
|
||||
"claude-2.0": 0.5138929040607993,
|
||||
"claude-3-5-sonnet-20240620": 0.5002552323803292,
|
||||
"claude-3-opus-20240229": 0.5061579533816516,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.38599872196923046,
|
||||
"gemini-1.5-pro-exp-0801": 0.4303694115381003,
|
||||
"gemini-advanced-0514": 0.4879802638437957,
|
||||
"gemini-pro-dev-api": 0.5098995340168142,
|
||||
"glm-4-0520": 0.5199790593996516,
|
||||
"gpt-3.5-turbo-0314": 0.4643874700255357,
|
||||
"gpt-4o-2024-05-13": 0.4891547255096569,
|
||||
"gpt-4o-2024-08-06": 0.4907702429276354,
|
||||
"gpt-4o-mini-2024-07-18": 0.5,
|
||||
"llama-2-70b-chat": 0.5266731022503263,
|
||||
"llama-3.1-70b-instruct": 0.4882504047493449,
|
||||
"nemotron-4-340b-instruct": 0.5218711102154154,
|
||||
"yi-large-preview": 0.5263558063552674
|
||||
},
|
||||
"llama-2-70b-chat": {
|
||||
"athene-70b-0725": 0.4811195552160165,
|
||||
"bard-jan-24-gemini-pro": 0.4518289406610546,
|
||||
"chatgpt-4o-latest": 0.40606343428200253,
|
||||
"claude-1": 0.4539411733076995,
|
||||
"claude-2.0": 0.4872008300180327,
|
||||
"claude-3-5-sonnet-20240620": 0.4735814107159531,
|
||||
"claude-3-opus-20240229": 0.4794713636760069,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3610160908710415,
|
||||
"gemini-1.5-pro-exp-0801": 0.40440647883548325,
|
||||
"gemini-advanced-0514": 0.4613567183023448,
|
||||
"gemini-pro-dev-api": 0.4832086967057715,
|
||||
"glm-4-0520": 0.4932916575528028,
|
||||
"gpt-3.5-turbo-0314": 0.43795013162446517,
|
||||
"gpt-4o-2024-05-13": 0.4625249859152287,
|
||||
"gpt-4o-2024-08-06": 0.46413246105753847,
|
||||
"gpt-4o-mini-2024-07-18": 0.47332689774967374,
|
||||
"llama-2-70b-chat": 0.5,
|
||||
"llama-3.1-70b-instruct": 0.46162540860366846,
|
||||
"nemotron-4-340b-instruct": 0.49518677639716263,
|
||||
"yi-large-preview": 0.49968180936417794
|
||||
},
|
||||
"llama-3.1-70b-instruct": {
|
||||
"athene-70b-0725": 0.5195508071470352,
|
||||
"bard-jan-24-gemini-pro": 0.4901305556202252,
|
||||
"chatgpt-4o-latest": 0.4436251505621573,
|
||||
"claude-1": 0.4922610506835726,
|
||||
"claude-2.0": 0.5256257671206118,
|
||||
"claude-3-5-sonnet-20240620": 0.512004683628698,
|
||||
"claude-3-opus-20240229": 0.5179023674394149,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3971975143004503,
|
||||
"gemini-1.5-pro-exp-0801": 0.4419289678886945,
|
||||
"gemini-advanced-0514": 0.49972970640339814,
|
||||
"gemini-pro-dev-api": 0.5216390614329225,
|
||||
"glm-4-0520": 0.531698889917426,
|
||||
"gpt-3.5-turbo-0314": 0.47609705817543935,
|
||||
"gpt-4o-2024-05-13": 0.5009047819370231,
|
||||
"gpt-4o-2024-08-06": 0.5025209317172257,
|
||||
"gpt-4o-mini-2024-07-18": 0.5117495952506551,
|
||||
"llama-2-70b-chat": 0.5383745913963316,
|
||||
"llama-3.1-70b-instruct": 0.5,
|
||||
"nemotron-4-340b-instruct": 0.5335861820021834,
|
||||
"yi-large-preview": 0.5380582595922262
|
||||
},
|
||||
"nemotron-4-340b-instruct": {
|
||||
"athene-70b-0725": 0.4859276634793583,
|
||||
"bard-jan-24-gemini-pro": 0.45660191547771745,
|
||||
"chatgpt-4o-latest": 0.4107151817570633,
|
||||
"claude-1": 0.458717789260918,
|
||||
"claude-2.0": 0.4920120852303797,
|
||||
"claude-3-5-sonnet-20240620": 0.47838363949618123,
|
||||
"claude-3-opus-20240229": 0.4842783735313118,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3654693315901167,
|
||||
"gemini-1.5-pro-exp-0801": 0.4090523175971309,
|
||||
"gemini-advanced-0514": 0.4661447537705109,
|
||||
"gemini-pro-dev-api": 0.4880180467685175,
|
||||
"glm-4-0520": 0.49810463635993246,
|
||||
"gpt-3.5-turbo-0314": 0.4426948962831791,
|
||||
"gpt-4o-2024-05-13": 0.4673146269400416,
|
||||
"gpt-4o-2024-08-06": 0.46892422516162097,
|
||||
"gpt-4o-mini-2024-07-18": 0.47812888978458457,
|
||||
"llama-2-70b-chat": 0.5048132236028374,
|
||||
"llama-3.1-70b-instruct": 0.4664138179978165,
|
||||
"nemotron-4-340b-instruct": 0.5,
|
||||
"yi-large-preview": 0.5044950605041638
|
||||
},
|
||||
"yi-large-preview": {
|
||||
"athene-70b-0725": 0.48143729978415806,
|
||||
"bard-jan-24-gemini-pro": 0.4521441972422958,
|
||||
"chatgpt-4o-latest": 0.4063704306655477,
|
||||
"claude-1": 0.45425668238107064,
|
||||
"claude-2.0": 0.48751881733159297,
|
||||
"claude-3-5-sonnet-20240620": 0.4738987237070381,
|
||||
"claude-3-opus-20240229": 0.47978902623790215,
|
||||
"gemini-1.5-pro-api-0409-preview": 0.3613097481008948,
|
||||
"gemini-1.5-pro-exp-0801": 0.40471307611584606,
|
||||
"gemini-advanced-0514": 0.4616730238748474,
|
||||
"gemini-pro-dev-api": 0.48352653528154693,
|
||||
"glm-4-0520": 0.4936097936282313,
|
||||
"gpt-3.5-turbo-0314": 0.4382634466315249,
|
||||
"gpt-4o-2024-05-13": 0.462841404205153,
|
||||
"gpt-4o-2024-08-06": 0.46444902876094657,
|
||||
"gpt-4o-mini-2024-07-18": 0.4736441936447327,
|
||||
"llama-2-70b-chat": 0.5003181906358221,
|
||||
"llama-3.1-70b-instruct": 0.4619417404077737,
|
||||
"nemotron-4-340b-instruct": 0.4955049394958363,
|
||||
"yi-large-preview": 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 226 KiB |
@@ -0,0 +1,66 @@
|
||||
"""Fail-closed verifier for a saved Experiment 7-7 run."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(8 * 1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("run_dir", type=Path)
|
||||
parser.add_argument("--input", type=Path, help="Optionally re-hash the 2 GB Arena input")
|
||||
args = parser.parse_args()
|
||||
|
||||
manifest_path = args.run_dir / "manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
failures: list[str] = []
|
||||
if manifest.get("experiment") != "7-7":
|
||||
failures.append("wrong experiment id")
|
||||
if manifest.get("status") != "passed" or manifest.get("official_complete") is not True:
|
||||
failures.append("run is not officially complete")
|
||||
false_gates = sorted(name for name, passed in manifest.get("gates", {}).items() if passed is not True)
|
||||
if false_gates:
|
||||
failures.append(f"false gates: {false_gates}")
|
||||
|
||||
for name, expected in manifest.get("artifacts", {}).items():
|
||||
path = args.run_dir / name
|
||||
if not path.is_file():
|
||||
failures.append(f"missing artifact: {name}")
|
||||
continue
|
||||
if path.stat().st_size != expected.get("bytes"):
|
||||
failures.append(f"size mismatch: {name}")
|
||||
if sha256_file(path) != expected.get("sha256"):
|
||||
failures.append(f"sha256 mismatch: {name}")
|
||||
|
||||
project = Path(__file__).resolve().parents[1]
|
||||
for name, expected_hash in manifest.get("sources", {}).items():
|
||||
path = project / name
|
||||
if not path.is_file() or sha256_file(path) != expected_hash:
|
||||
failures.append(f"source mismatch: {name}")
|
||||
|
||||
if args.input:
|
||||
expected = manifest["input"]
|
||||
if args.input.stat().st_size != expected["bytes"]:
|
||||
failures.append("input size mismatch")
|
||||
if sha256_file(args.input) != expected["sha256"]:
|
||||
failures.append("input sha256 mismatch")
|
||||
|
||||
result = {"valid": not failures, "failures": failures}
|
||||
print(json.dumps(result, indent=2))
|
||||
if failures:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user