ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter7/elo-leaderboard/benchmark.py

@@ -0,0 +1,144 @@
+"""
+Benchmark script to compare performance of different Elo implementations
+"""
+import time
+import pandas as pd
+import numpy as np
+from elo_rating import EloRatingSystem
+from optimized_elo import build_leaderboard_optimized
+from data_loader import load_arena_data, filter_data
+
+
+def benchmark_basic_elo(df: pd.DataFrame) -> float:
+    """Benchmark the basic Elo implementation."""
+    print("\n" + "="*80)
+    print("Benchmarking Basic Elo Implementation (Python dict)")
+    print("="*80)
+    
+    start_time = time.time()
+    
+    elo = EloRatingSystem(initial_rating=1000.0, k_factor=32.0)
+    
+    for _, row in df.iterrows():
+        elo.update_ratings(row['model_a'], row['model_b'], row['winner'])
+    
+    end_time = time.time()
+    elapsed = end_time - start_time
+    
+    leaderboard = elo.get_leaderboard()
+    
+    print(f"✓ Processed {len(df)} matches in {elapsed:.2f} seconds")
+    print(f"  Speed: {len(df)/elapsed:.0f} matches/second")
+    print(f"  Top 3 models: {[m[0] for m in leaderboard[:3]]}")
+    
+    return elapsed
+
+
+def benchmark_optimized_elo(df: pd.DataFrame) -> float:
+    """Benchmark the NumPy + Numba optimized implementation."""
+    print("\n" + "="*80)
+    print("Benchmarking Optimized Elo Implementation (NumPy + Numba JIT)")
+    print("="*80)
+    
+    start_time = time.time()
+    
+    elo = build_leaderboard_optimized(
+        df,
+        initial_rating=1000.0,
+        k_factor=32.0,
+        show_progress=False
+    )
+    
+    end_time = time.time()
+    elapsed = end_time - start_time
+    
+    leaderboard = elo.get_leaderboard()
+    
+    print(f"✓ Processed {len(df)} matches in {elapsed:.2f} seconds")
+    print(f"  Speed: {len(df)/elapsed:.0f} matches/second")
+    print(f"  Top 3 models: {[m[0] for m in leaderboard[:3]]}")
+    
+    return elapsed
+
+
+def main():
+    """Run benchmark comparison."""
+    print("="*80)
+    print("ELO RATING COMPUTATION BENCHMARK")
+    print("="*80)
+    print("\nThis benchmark compares the performance of different Elo implementations")
+    print("on Chatbot Arena voting data.\n")
+    
+    # Load data
+    print("Loading data...")
+    try:
+        df = load_arena_data("arena_data.json")
+    except FileNotFoundError:
+        print("Error: arena_data.json not found. Please run main.py first to download the data.")
+        return
+    
+    # Filter for blind votes
+    print("Filtering data...")
+    df_filtered = filter_data(df, anony_only=True, min_turn=1)
+    
+    # Use a subset for quick benchmarking (can change to full dataset)
+    sample_size = 50000
+    if len(df_filtered) > sample_size:
+        print(f"\nUsing a sample of {sample_size} matches for benchmarking")
+        print("(To benchmark on full dataset, set sample_size = len(df_filtered))")
+        df_sample = df_filtered.head(sample_size).copy()
+    else:
+        df_sample = df_filtered.copy()
+    
+    print(f"\nBenchmark dataset: {len(df_sample)} matches")
+    print(f"Unique models: {len(set(df_sample['model_a'].unique()) | set(df_sample['model_b'].unique()))}")
+    
+    # Warm up Numba JIT (first run compiles the functions)
+    print("\n" + "-"*80)
+    print("Warming up Numba JIT compiler (first run)...")
+    print("-"*80)
+    df_tiny = df_sample.head(1000)
+    build_leaderboard_optimized(df_tiny, show_progress=False)
+    print("✓ JIT compilation complete")
+    
+    # Run benchmarks
+    time_basic = benchmark_basic_elo(df_sample)
+    time_optimized = benchmark_optimized_elo(df_sample)
+    
+    # Results summary
+    print("\n" + "="*80)
+    print("BENCHMARK RESULTS")
+    print("="*80)
+    
+    speedup = time_basic / time_optimized if time_optimized > 0 else 0
+    
+    print(f"\nBasic Implementation:     {time_basic:8.2f} seconds")
+    print(f"Optimized Implementation: {time_optimized:8.2f} seconds")
+    print(f"\nSpeedup: {speedup:.1f}x faster")
+    pct_reduction = (1 - time_optimized / time_basic) * 100 if time_basic > 0 else 0.0
+    print(f"Time saved: {time_basic - time_optimized:.2f} seconds ({pct_reduction:.1f}% reduction)")
+    
+    # Extrapolate to full dataset
+    if len(df_sample) > 0 and len(df_sample) < len(df_filtered):
+        full_time_basic = time_basic * (len(df_filtered) / len(df_sample))
+        full_time_optimized = time_optimized * (len(df_filtered) / len(df_sample))
+        
+        print(f"\nExtrapolated times for full dataset ({len(df_filtered)} matches):")
+        print(f"  Basic:     ~{full_time_basic/60:.1f} minutes")
+        print(f"  Optimized: ~{full_time_optimized/60:.1f} minutes")
+        print(f"  Time saved: ~{(full_time_basic - full_time_optimized)/60:.1f} minutes")
+    
+    print("\n" + "="*80)
+    print("\nOptimization Techniques Applied:")
+    print("  • NumPy arrays instead of Python dicts (O(1) integer indexing)")
+    print("  • Numba JIT compilation (compiles hot loops to machine code)")
+    print("  • Pre-allocated arrays (no dynamic memory allocation)")
+    print("  • Integer model indices (no string lookups)")
+    print("  • Vectorized operations where possible")
+    print("\nFor the full optimized pipeline with parallel processing, run main_optimized.py")
+    print("="*80 + "\n")
+
+
+if __name__ == "__main__":
+    main()
+