{ "schema_version": "2.0", "experiment": "7-11", "status": "smoke", "generated_at_utc": "2026-07-31T17:53:12Z", "command": "experiment.py 7-11 --config default_config.yaml --test-id layer1_01_bank_account --readiness results/full_matrix_backend_readiness_20260731.json --output results/live_7_11_matrix_layer1.json", "run_scope": { "suite_case_count": 60, "requested_case_count": 1, "requested_test_ids": [ "layer1_01_bank_account" ], "configured_trajectory_count": 24, "completed_trajectory_count": 24, "error_trajectory_count": 0, "all_configured_trajectories_completed": true, "full_60_case_suite_completed": false, "trajectory_matrix_complete": false, "cost_accounting_complete": true, "validation_scope": "smoke" }, "completion": { "status": "smoke", "evidence_complete": false, "trajectory_matrix_complete": false, "cost_accounting_complete": true, "real_api_evidence_only": true, "backend_readiness_complete": true, "expected_case_count": 60, "observed_case_count": 1, "expected_cells_per_case": 24, "expected_full_trajectory_count": 1440, "configured_trajectory_count": 24, "completed_trajectory_count": 24, "error_trajectory_count": 0, "matrix_shape": { "embeddings": 4, "rerankers": 3, "main_models": 2, "cells_per_case": 24 }, "duplicate_cells": [], "missing_cells_for_observed_cases": [], "unexpected_cells": [], "blockers": [ { "code": "missing_cases", "message": "1/60 cases are present" } ] }, "configuration": { "chat_models": { "kimi": { "model": "kimi-k2.5", "base_url": "https://api.moonshot.cn/v1", "api_key_env": "KIMI_API_KEY", "disable_thinking": true, "temperature": 0.6, "pricing": { "currency": "CNY", "as_of_date": "2026-07-29", "source_url": "https://platform.kimi.com/docs/pricing/chat-k25", "input_per_million": 4.0, "cached_input_per_million": 0.7, "output_per_million": 21.0, "source_note": "Published Kimi K2.5 list price; input rate is cache-miss/uncached." } }, "doubao": { "model": "doubao-seed-1-6-250615", "base_url": "https://ark.cn-beijing.volces.com/api/v3", "api_key_env": "ARK_API_KEY", "pricing": { "currency": "CNY", "as_of_date": "2026-07-31", "source_url": "https://www.volcengine.com/docs/82379/1544106", "input_per_million": 0.8, "output_per_million": 8.0, "source_note": "Published Doubao-Seed-1.6 list price for the 0-32K input-length range (input 0.8 CNY/M, output 8 CNY/M). All matrix prompts are below 32K." } }, "openai-mini": { "model": "gpt-4.1-mini", "base_url": "https://api.openai.com/v1", "api_key_env": "OPENAI_API_KEY" } }, "embeddings": { "bge-m3": { "model": "baai/bge-m3", "base_url": "https://openrouter.ai/api/v1", "api_key_env": "OPENROUTER_API_KEY", "pricing": { "currency": "USD", "as_of_date": "2026-07-31", "source_url": "https://openrouter.ai/baai/bge-m3", "input_per_million": 0.01, "source_note": "OpenRouter catalog price for baai/bge-m3; matches the per-token cost returned by the /api/v1/embeddings/models listing and live usage." } }, "openai-small": { "model": "openai/text-embedding-3-small", "base_url": "https://openrouter.ai/api/v1", "api_key_env": "OPENROUTER_API_KEY", "pricing": { "currency": "USD", "as_of_date": "2026-07-31", "source_url": "https://openrouter.ai/openai/text-embedding-3-small", "input_per_million": 0.02, "source_note": "OpenRouter catalog price, identical to the published text-embedding-3-small list price." } }, "qwen3": { "model": "qwen/qwen3-embedding-8b", "base_url": "https://openrouter.ai/api/v1", "api_key_env": "OPENROUTER_API_KEY", "pricing": { "currency": "USD", "as_of_date": "2026-07-31", "source_url": "https://openrouter.ai/qwen/qwen3-embedding-8b", "input_per_million": 0.01, "source_note": "OpenRouter catalog price for qwen/qwen3-embedding-8b." } }, "mistral": { "model": "mistral-embed", "base_url": "https://api.mistral.ai/v1", "api_key_env": "MISTRAL_API_KEY", "pricing": { "currency": "USD", "as_of_date": "2026-07-29", "source_url": "https://mistral.ai/pricing/api/", "input_per_million": 0.1, "source_note": "Published Mistral Embed list price." } } }, "rerankers": { "none": { "type": "none" }, "doubao-semantic": { "type": "llm", "chat_model": "doubao" }, "kimi-semantic": { "type": "llm", "chat_model": "kimi" } }, "judge": { "evaluator": "kimi", "model": "kimi-k2.5" }, "experiment_7_4": { "main_model": "kimi", "embedding": "mistral", "reranker": "kimi-semantic", "rounds_per_chunk": 6, "overlap": 2 }, "experiment_7_11": { "embeddings": [ "bge-m3", "openai-small", "qwen3", "mistral" ], "rerankers": [ "none", "doubao-semantic", "kimi-semantic" ], "main_models": [ "kimi", "doubao" ], "retrieval_judge_model": "kimi", "max_search_rounds": 3, "rounds_per_chunk": 6, "overlap": 2 }, "execution_readiness": { "source_file": "results/full_matrix_backend_readiness_20260731.json", "generated_at_utc": "2026-07-31T17:36:29Z", "all_required_backends_ready": true, "execution_config_fingerprint": "52dda0533016f6c980fc1922ee0d8a5822e671976a2cac5c7191219da97b4b4a", "validated": true } }, "pricing_manifest": [ { "component": "chat", "name": "doubao", "model": "doubao-seed-1-6-250615", "pricing_via": null, "pricing": { "currency": "CNY", "as_of_date": "2026-07-31", "source_url": "https://www.volcengine.com/docs/82379/1544106", "input_per_million": 0.8, "output_per_million": 8.0, "source_note": "Published Doubao-Seed-1.6 list price for the 0-32K input-length range (input 0.8 CNY/M, output 8 CNY/M). All matrix prompts are below 32K." }, "status": "validated" }, { "component": "chat", "name": "kimi", "model": "kimi-k2.5", "pricing_via": null, "pricing": { "currency": "CNY", "as_of_date": "2026-07-29", "source_url": "https://platform.kimi.com/docs/pricing/chat-k25", "input_per_million": 4.0, "cached_input_per_million": 0.7, "output_per_million": 21.0, "source_note": "Published Kimi K2.5 list price; input rate is cache-miss/uncached." }, "status": "validated" }, { "component": "embedding", "name": "bge-m3", "model": "baai/bge-m3", "pricing_via": null, "pricing": { "currency": "USD", "as_of_date": "2026-07-31", "source_url": "https://openrouter.ai/baai/bge-m3", "input_per_million": 0.01, "source_note": "OpenRouter catalog price for baai/bge-m3; matches the per-token cost returned by the /api/v1/embeddings/models listing and live usage." }, "status": "validated" }, { "component": "embedding", "name": "mistral", "model": "mistral-embed", "pricing_via": null, "pricing": { "currency": "USD", "as_of_date": "2026-07-29", "source_url": "https://mistral.ai/pricing/api/", "input_per_million": 0.1, "source_note": "Published Mistral Embed list price." }, "status": "validated" }, { "component": "embedding", "name": "openai-small", "model": "openai/text-embedding-3-small", "pricing_via": null, "pricing": { "currency": "USD", "as_of_date": "2026-07-31", "source_url": "https://openrouter.ai/openai/text-embedding-3-small", "input_per_million": 0.02, "source_note": "OpenRouter catalog price, identical to the published text-embedding-3-small list price." }, "status": "validated" }, { "component": "embedding", "name": "qwen3", "model": "qwen/qwen3-embedding-8b", "pricing_via": null, "pricing": { "currency": "USD", "as_of_date": "2026-07-31", "source_url": "https://openrouter.ai/qwen/qwen3-embedding-8b", "input_per_million": 0.01, "source_note": "OpenRouter catalog price for qwen/qwen3-embedding-8b." }, "status": "validated" }, { "component": "reranker", "name": "doubao-semantic", "model": "doubao-seed-1-6-250615", "pricing_via": "chat_models.doubao", "pricing": { "currency": "CNY", "as_of_date": "2026-07-31", "source_url": "https://www.volcengine.com/docs/82379/1544106", "input_per_million": 0.8, "output_per_million": 8.0, "source_note": "Published Doubao-Seed-1.6 list price for the 0-32K input-length range (input 0.8 CNY/M, output 8 CNY/M). All matrix prompts are below 32K." }, "status": "validated" }, { "component": "reranker", "name": "kimi-semantic", "model": "kimi-k2.5", "pricing_via": "chat_models.kimi", "pricing": { "currency": "CNY", "as_of_date": "2026-07-29", "source_url": "https://platform.kimi.com/docs/pricing/chat-k25", "input_per_million": 4.0, "cached_input_per_million": 0.7, "output_per_million": 21.0, "source_note": "Published Kimi K2.5 list price; input rate is cache-miss/uncached." }, "status": "validated" }, { "component": "reranker", "name": "none", "model": null, "pricing_via": null, "pricing": null, "status": "not_applicable_zero_cost" } ], "pricing_coverage": { "completed_trajectory_count": 24, "observed_token_count": 242086, "observed_primary_token_count": 183886, "observed_fixed_query_token_count": 58200, "legacy_fixed_query_unpriced_token_lower_bound": 0, "unpriced_token_count": 0, "unpriced_request_count": 0, "all_observed_usage_priced": true, "total_cost_by_currency": { "CNY": 0.4622971, "USD": 0.0028121000000000005 }, "usd_total_without_currency_conversion": 0.0028121000000000005, "currency_conversion_applied": false, "note": "Native-currency totals are intentionally not combined. RMB/CNY is never converted to USD without an explicit dated FX source and rate in the report. Schema-1 fixed-query token totals are a lower bound when only their unpriced count was retained." }, "records": [ { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "bge-m3", "reranker": "none", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 4872.473999999784, "cost_usd": 2.9710000000000002e-05, "input_tokens": 4937, "output_tokens": 79, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.009522999999999998, "USD": 2.9710000000000002e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 0.5, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 1986.3613750003424, "embedding_index_cost_usd": 2.964e-05, "embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "fixed_query_retrieval_latency_ms": 396.5964580002037, "fixed_query_retrieval_cost_usd": 2.2e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-07 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "3e0cdf40421392c0", "8ca88f6573db460e", "15d7bbeb79e1d7d2", "4e4ffea07db242b6", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identifies the exact checking account number 4429853327 without any digit errors. The agent also correctly provides the routing number 123006800. There is no confusion with the savings account number 4429853328 or any other numbers in the conversation (SSN 547-82-9163, phone 503-555-8924, etc.).", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328." ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent retrieves both pieces of information necessary for the user's stated purpose: the checking account number for direct deposit setup, and proactively includes the routing number which is required for direct deposit. The evaluation criteria specifically notes that 'the agent should ideally also mention the routing number 123006800 which was discussed in that context.'", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Yes, my paycheck from TechCorp." ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly links the user's question about direct deposit setup to the relevant context in the conversation where this was discussed. The agent understands that routing numbers are needed for direct deposit and retrieves both pieces of information from the same contextual exchange. No ambiguity or conflicts exist to resolve.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Yes, my paycheck from TechCorp." ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrates excellent proactivity by including the routing number 123006800, which is essential for direct deposit setup, even though the user only asked for the account number. The evaluation criteria explicitly states this as the 'excellent example' for proactivity: 'includes the remembered routing number when direct-deposit setup makes it useful.' The agent correctly inferred from 'set up my direct deposit at work' that both numbers would be needed.", "evidence": [ "User Question: What was my checking account number again? I need it to set up my direct deposit at work.", "Agent Response: your checking account number is 4429853327... The routing number for direct deposit is 123006800.", "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327." ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims in the agent response are directly supported by the authoritative source: checking account number 4429853327, routing number 123006800, First National Bank as the institution, and Premium Checking account type." ], "reasoning": "No material factual claims are unsupported or contradicted by the source. The agent's response is fully grounded in the conversation transcript." }, "answer": "Based on your conversation history from November 15th, 2024, your checking account number is **4429853327**.\n\nThis was set up with First National Bank when you opened your Premium Checking account. The routing number for direct deposit is **123006800**.", "trace": [ { "event": "search_memory", "query": "checking account number direct deposit", "chunk_ids": [ "3e0cdf40421392c0", "8ca88f6573db460e", "15d7bbeb79e1d7d2", "4e4ffea07db242b6", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrates perfect performance on this memory retrieval task. It correctly identifies the specific checking account number requested, distinguishes it from the savings account number, and proactively includes the routing number which is necessary for the user's stated purpose of setting up direct deposit. All information is factually accurate per the source conversation.", "evaluation_suggestions": "No improvements needed. The response is a model example of accurate memory retrieval with appropriate proactive assistance.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "bge-m3", "reranker": "none", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 15984.19020900019, "cost_usd": 2.9690000000000002e-05, "input_tokens": 5779, "output_tokens": 476, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.006056000000000001, "USD": 2.9690000000000002e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 0.5, "retrieval_mrr": 0.5, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 1986.3613750003424, "embedding_index_cost_usd": 2.964e-05, "embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "fixed_query_retrieval_latency_ms": 396.5964580002037, "fixed_query_retrieval_cost_usd": 2.2e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-07 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "15d7bbeb79e1d7d2", "8ca88f6573db460e", "4e4ffea07db242b6", "3e0cdf40421392c0", "c466f611e5375254" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provided the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguished this from the savings account number 4429853328, providing only the checking account number as requested. The routing number 123006800 was also stated exactly as provided in the source.", "evidence": [ "assistant: ...your new checking account number is 4429853327", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328", "assistant: Our routing number is 123006800" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent fully recalled both the checking account number (4429853327) and the routing number (123006800) that were necessary for the user's direct deposit setup. The evaluation criteria specifically noted that 'the agent should ideally also mention the routing number 123006800 which was discussed in that context,' and the agent did so.", "evidence": [ "assistant: Our routing number is 123006800, and your new checking account number is 4429853327", "Agent response: Your checking account number is 4429853327. You can use this along with the routing number 123006800 to set up direct deposit for your TechCorp paycheck." ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly linked the user's request for the account number 'to set up my direct deposit at work' with the prior context where direct deposit was discussed for the TechCorp paycheck. The agent appropriately connected these two conversation threads and understood the purpose of the request, adding relevant context (TechCorp employer, routing number) that was specifically tied to the direct deposit setup scenario.", "evidence": [ "user: Yes, my paycheck from TechCorp", "assistant: I can provide you with our routing number and your new account number for that", "User question: What was my checking account number again? I need it to set up my direct deposit at work", "Agent response: ...to set up direct deposit for your TechCorp paycheck" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrated excellent proactivity by not only providing the requested checking account number but also anticipating the complete needs for direct deposit setup. Per the evaluation criteria's 'excellent example,' the agent 'includes the remembered routing number when direct-deposit setup makes it useful.' The agent also helpfully reminded the user of their employer (TechCorp), reinforcing the correct context for the direct deposit. This was safe, useful, and directly relevant next-step help.", "evidence": [ "Agent response: Your checking account number is 4429853327. You can use this along with the routing number 123006800 to set up direct deposit for your TechCorp paycheck." ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims in the agent response are directly supported by the authoritative source: checking account number 4429853327, routing number 123006800, and TechCorp as employer for direct deposit." ], "reasoning": "No material factual claims in the agent's response are unsupported or contradicted by the authoritative conversation source. The account number, routing number, and employer reference are all verbatim or directly inferable from the source conversation." }, "answer": "Your checking account number is 4429853327. You can use this along with the routing number 123006800 to set up direct deposit for your TechCorp paycheck.", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "15d7bbeb79e1d7d2", "8ca88f6573db460e", "4e4ffea07db242b6", "3e0cdf40421392c0", "c466f611e5375254" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response demonstrates perfect memory retrieval and contextual understanding. It provides the exact checking account number requested, correctly distinguishes it from the savings account number, and proactively includes the routing number and employer context that were specifically tied to the direct deposit discussion in the original conversation. This matches the expected behavior described in the evaluation criteria.", "evaluation_suggestions": "No improvements needed. The response is exemplary for this memory retrieval task.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "bge-m3", "reranker": "doubao-semantic", "main_model": "kimi", "success": true, "reward": 0.916667, "steps": 2, "tool_calls": 1, "latency_ms": 33213.47191800032, "cost_usd": 2.9710000000000002e-05, "input_tokens": 7506, "output_tokens": 1059, "unpriced_tokens": 0, "cached_input_tokens": 161, "cost_by_currency": { "CNY": 0.015945099999999997, "USD": 2.9710000000000002e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 1986.3613750003424, "embedding_index_cost_usd": 2.964e-05, "embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "fixed_query_retrieval_latency_ms": 16083.018499999525, "fixed_query_retrieval_cost_usd": 2.2e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-07, "CNY": 0.0078184 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 629, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 3, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provides the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguishes this from the savings account number 4429853328, which was mentioned in the same conversation context. No numbers are confused or misstated.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328.", "Agent Response: Your checking account number is **4429853327**." ], "boundary_case": null }, "recall": { "grade": "good", "score": 3, "reasoning": "The agent successfully recalls and provides the core requested information (checking account number 4429853327). The agent also includes the routing number 123006800, which is relevant to the user's stated purpose of setting up direct deposit. However, the agent omits that the savings account number is 4429853328, which while not explicitly requested, is part of the complete account information set. This is a minor omission as the user specifically asked for the checking account number only.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "Agent Response: Your checking account number is **4429853327**. This was provided to you when you set up your Premium Checking account with First National Bank on November 15th, along with routing number 123006800 for your direct deposit setup." ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identifies that the user needs the checking account number specifically for direct deposit setup, and proactively includes the routing number which is necessary for that purpose. The agent correctly links the account number to the correct account type (Premium Checking) and the correct context (direct deposit setup with TechCorp). No ambiguity or confusion exists in the response.", "evidence": [ "user: Yes, my paycheck from TechCorp.", "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "User Question: What was my checking account number again? I need it to set up my direct deposit at work.", "Agent Response: This was provided to you when you set up your Premium Checking account with First National Bank on November 15th, along with routing number 123006800 for your direct deposit setup." ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrates excellent proactivity by including the routing number 123006800, which is essential information for setting up direct deposit, even though the user only explicitly asked for the account number. The agent correctly inferred from the user's context ('I need it to set up my direct deposit at work') that both pieces of information would be useful. This is exactly the type of contextual awareness described in the proactivity criteria.", "evidence": [ "User Question: What was my checking account number again? I need it to set up my direct deposit at work.", "Agent Response: This was provided to you when you set up your Premium Checking account with First National Bank on November 15th, along with routing number 123006800 for your direct deposit setup.", "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327." ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims in the agent response are supported by the authoritative source: checking account number 4429853327, routing number 123006800, Premium Checking account type, November 15th date, and direct deposit context." ], "reasoning": "The agent response contains no material factual claims that are unsupported or contradicted by the authoritative conversation source. The account number, routing number, account type, date, and purpose context are all directly verifiable from the source conversation." }, "answer": "Your checking account number is **4429853327**. \n\nThis was provided to you when you set up your Premium Checking account with First National Bank on November 15th, along with routing number 123006800 for your direct deposit setup.", "trace": [ { "event": "search_memory", "query": "checking account number direct deposit", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrates strong memory retrieval capabilities by accurately providing the checking account number 4429853327 and proactively including the routing number 123006800 based on the user's stated need for direct deposit setup. The response shows excellent precision with exact number recall, good recall with all necessary information present, excellent reasoning in linking the context correctly, and excellent proactivity in anticipating the user's complete needs. No hallucinations are detected.", "evaluation_suggestions": "No significant improvements needed. The response is accurate and appropriately proactive. If desired, the agent could optionally remind the user of their savings account number (4429853328) for completeness, though this was not requested and would be a minor enhancement rather than a correction.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "bge-m3", "reranker": "doubao-semantic", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 25902.4385849998, "cost_usd": 2.9690000000000002e-05, "input_tokens": 8351, "output_tokens": 891, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.011433600000000002, "USD": 2.9690000000000002e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 1986.3613750003424, "embedding_index_cost_usd": 2.964e-05, "embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "fixed_query_retrieval_latency_ms": 16083.018499999525, "fixed_query_retrieval_cost_usd": 2.2e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-07, "CNY": 0.0078184 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 629, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provides the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguishes this from the savings account number 4429853328 and does not confuse it with any other numbers in the conversation (SSN 547-82-9163, phone 503-555-8924, Wells Fargo account 8847293001, etc.). The routing number 123006800 is also stated correctly.", "evidence": [ "Your checking account number is 4429853327", "routing number (123006800)", "your new checking account number is 4429853327", "Correct. And your savings account number is 4429853328" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent retrieves all necessary information for the question: the specific checking account number requested. Additionally, the agent proactively recalls the routing number which is necessary for direct deposit setup, exactly as specified in the evaluation criteria. The agent also correctly recalls the context that this was for TechCorp direct deposit setup.", "evidence": [ "Your checking account number is 4429853327", "when you set up direct deposit for your TechCorp paycheck", "routing number (123006800) as well for the direct deposit form" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identifies that the user is asking about direct deposit setup (mentioned in their question) and links this to the appropriate context from the conversation. The agent resolves the potential ambiguity between checking and savings accounts by providing only the checking account number as requested, while also offering the routing number which is specifically needed for direct deposit forms. No cross-session links or temporal conflicts exist in this single-conversation test case.", "evidence": [ "I need it to set up my direct deposit at work", "when you set up direct deposit for your TechCorp paycheck", "routing number (123006800) as well for the direct deposit form" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrates excellent proactivity by anticipating that the routing number will also be needed for the direct deposit form and offering it proactively. This directly matches the evaluation criteria's 'excellent example' of 'includes the remembered routing number when direct-deposit setup makes it useful.' The agent correctly interprets the user's implicit need based on their stated purpose (direct deposit setup).", "evidence": [ "Let me know if you need the routing number (123006800) as well for the direct deposit form" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "Your checking account number is 4429853327", "routing number (123006800)", "when you set up direct deposit for your TechCorp paycheck" ], "reasoning": "All factual claims in the agent's response are directly supported by the authoritative conversation source. The checking account number 4429853327, routing number 123006800, and the context of TechCorp direct deposit setup are all explicitly stated in the source conversation. No invented details or contradictions are present." }, "answer": "Your checking account number is 4429853327. This was confirmed during your account setup on November 15, 2024, when you set up direct deposit for your TechCorp paycheck. Let me know if you need the routing number (123006800) as well for the direct deposit form.", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response demonstrates perfect memory retrieval and appropriate contextual awareness. It provides the exact checking account number requested, correctly distinguishes it from the savings account, and proactively offers the routing number which is necessary for direct deposit setup. All information is factually accurate per the authoritative source, with no hallucinations or errors.", "evaluation_suggestions": "No improvements needed. The response is exemplary for this memory retrieval task.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "bge-m3", "reranker": "kimi-semantic", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 8581.173708001188, "cost_usd": 2.9710000000000002e-05, "input_tokens": 7797, "output_tokens": 157, "unpriced_tokens": 0, "cached_input_tokens": 929, "cost_by_currency": { "CNY": 0.0195353, "USD": 2.9710000000000002e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 1986.3613750003424, "embedding_index_cost_usd": 2.964e-05, "embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "fixed_query_retrieval_latency_ms": 3120.466540999587, "fixed_query_retrieval_cost_usd": 2.2e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-07, "CNY": 0.012367 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830", "3e0cdf40421392c0", "3a006babbe4e3fb2", "15d7bbeb79e1d7d2" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provided the exact checking account number 4429853327 without any digit errors. The agent also correctly provided the routing number 123006800. No confusion occurred with the savings account number 4429853328 or any other numbers in the conversation (SSN 547-82-9163, phone 503-555-8924, Wells Fargo account 8847293001, PIN 4827, etc.).", "evidence": [ "assistant: Our routing number is 123006800, and your new checking account number is 4429853327", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent retrieved both pieces of information necessary for the user's stated purpose: the checking account number (explicitly requested) and the routing number (needed for direct deposit setup, as the user mentioned). The agent correctly distinguished checking from savings and did not omit any required information.", "evidence": [ "assistant: Our routing number is 123006800, and your new checking account number is 4429853327", "user: Yes, my paycheck from TechCorp (indicating direct deposit need)" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly linked the user's question about 'direct deposit at work' to the specific context in the conversation where account numbers were provided for that exact purpose. The agent appropriately retrieved both the account number and routing number that were discussed in the direct deposit setup portion of the conversation, showing correct temporal and contextual linking.", "evidence": [ "user: Yes, my paycheck from TechCorp", "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrated excellent proactivity by anticipating that the routing number would be needed for direct deposit setup, even though the user only asked for the account number. This was directly relevant to the user's stated purpose ('set up my direct deposit at work') and matches the evaluation criteria's expected behavior of mentioning the routing number.", "evidence": [ "The routing number for your bank is **123006800** — you'll need both for setting up direct deposit at work" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims (checking account number 4429853327, routing number 123006800, bank name First National Bank, date November 15th) are directly supported by the authoritative source" ], "reasoning": "No material factual claims are unsupported or contradicted. The date 'November 15th' is correctly inferred from the conversation timestamp [2024-11-15 10:30:00]. The bank name 'First National Bank' appears multiple times in the source." }, "answer": "Your checking account number is **4429853327**.\n\nThis was provided when you opened your Premium Checking account at First National Bank on November 15th. The routing number for your bank is **123006800** — you'll need both for setting up direct deposit at work.", "trace": [ { "event": "search_memory", "query": "checking account number direct deposit", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830", "3e0cdf40421392c0", "3a006babbe4e3fb2", "15d7bbeb79e1d7d2" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response is fully accurate and complete. It correctly retrieves the checking account number 4429853327, distinguishes it from the savings account number 4429853328, and proactively includes the routing number 123006800 which is necessary for the user's stated purpose of setting up direct deposit. All information is directly supported by the authoritative conversation source.", "evaluation_suggestions": "No improvements needed. The response perfectly meets all evaluation criteria.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "bge-m3", "reranker": "kimi-semantic", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 16903.2389590011, "cost_usd": 2.9690000000000002e-05, "input_tokens": 7777, "output_tokens": 480, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.0173958, "USD": 2.9690000000000002e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 1986.3613750003424, "embedding_index_cost_usd": 2.964e-05, "embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "fixed_query_retrieval_latency_ms": 3120.466540999587, "fixed_query_retrieval_cost_usd": 2.2e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-07, "CNY": 0.012367 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provides the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguishes it from the savings account number 4429853328, which was mentioned in the same conversation context. No confusion with other numbers (SSN 547-82-9163, phone 503-555-8924, etc.) occurred.", "evidence": [ "Your checking account number is 4429853327", "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328." ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent successfully retrieves the core requested information (checking account number) and also includes the routing number 123006800, which was explicitly mentioned in the evaluation criteria as ideal for direct deposit setup context. Both pieces of information were discussed together in the original conversation when the user was setting up direct deposit for their TechCorp paycheck.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Yes, my paycheck from TechCorp." ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly links the user's question about direct deposit setup to the relevant context from the original conversation (TechCorp paycheck direct deposit setup). The agent appropriately connects the temporal sequence: the account number was provided specifically for direct deposit purposes during the original call, and the user is now following up on that same purpose. No ambiguity or conflicts exist in this single-session retrieval scenario.", "evidence": [ "user: Yes, my paycheck from TechCorp.", "assistant: I can provide you with our routing number and your new account number for that.", "This was confirmed during your account setup on November 15, 2024, when you set up direct deposit for your TechCorp paycheck" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrates excellent proactivity by anticipating the user's need for the routing number (123006800) given the stated purpose of setting up direct deposit at work. This is exactly the type of 'useful, safe next-step help' described in the rubric's excellent example. The agent offers this additional information without being explicitly asked, recognizing that both numbers are typically required for direct deposit forms.", "evidence": [ "Let me know if you need the routing number (123006800) as well for the direct deposit form.", "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims (4429853327, 123006800, November 15 2024, TechCorp) are directly supported by the authoritative source" ], "reasoning": "The agent's response contains no material factual claims absent from or contradicted by the authoritative conversation source. The account number, routing number, date, employer name, and context are all verbatim or directly inferable from the source. The phrase 'This was confirmed during your account setup on November 15, 2024' is a reasonable temporal inference from the conversation timestamp and confirmation exchange." }, "answer": "Your checking account number is 4429853327. This was confirmed during your account setup on November 15, 2024, when you set up direct deposit for your TechCorp paycheck. Let me know if you need the routing number (123006800) as well for the direct deposit form.", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrates perfect performance on this memory retrieval task. It accurately retrieves the specific checking account number requested, correctly distinguishes it from the savings account number, provides the routing number proactively given the direct deposit context, and maintains factual fidelity to the source conversation. No hallucinations, confusions, or omissions are present.", "evaluation_suggestions": "No improvements needed. The response is a model example of accurate memory retrieval with appropriate proactive assistance.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "openai-small", "reranker": "none", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 5347.5308330007465, "cost_usd": 5.6420000000000005e-05, "input_tokens": 4698, "output_tokens": 129, "unpriced_tokens": 0, "cached_input_tokens": 417, "cost_by_currency": { "CNY": 0.008840899999999999, "USD": 5.6420000000000005e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 0.5, "retrieval_mrr": 0.3333333333333333, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 0.5, "embedding_index_latency_ms": 1726.816417000009, "embedding_index_cost_usd": 5.632e-05, "embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "fixed_query_retrieval_latency_ms": 640.7745839997006, "fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 20, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "3e0cdf40421392c0", "3a006babbe4e3fb2", "8ca88f6573db460e", "4e4ffea07db242b6", "577692430b65653c" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identified the exact checking account number 4429853327 and distinguished it from the savings account (4429853328). The routing number 123006800 was also correctly provided. The agent also correctly noted the distinction from the previous Wells Fargo account (8847293001) without confusing any digits.", "evidence": [ "assistant: ...your new checking account number is 4429853327", "user: Let me write that down... 4429853327, right? assistant: Correct. And your savings account number is 4429853328", "Our routing number is 123006800" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent retrieved all necessary information: the checking account number (4429853327) as directly requested, and proactively included the routing number (123006800) which is required for direct deposit setup. The agent also correctly recalled the context that this was for TechCorp direct deposit setup.", "evidence": [ "assistant: ...your new checking account number is 4429853327", "Our routing number is 123006800", "user: Yes, my paycheck from TechCorp" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly understood the user's intent (direct deposit setup at work) and linked this to the previously discussed TechCorp paycheck direct deposit context. The agent properly distinguished between the new First National Bank checking account and the old Wells Fargo account, clarifying which account to use for the direct deposit.", "evidence": [ "Just to confirm - this was for the checking account you opened at the new bank (not your previous Wells Fargo account, which was 8847293001)" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent went beyond the minimal answer by including the routing number (123006800) which is essential for direct deposit setup, and asked a clarifying question to confirm the correct account is being used. This demonstrates excellent proactive assistance for the user's stated goal.", "evidence": [ "You'll also need the routing number for your direct deposit setup, which is 123006800", "Is this the account you want to use for your work direct deposit?" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims (4429853327, 123006800, 8847293001, TechCorp, MRobertson503 context) are directly supported by the authoritative source" ], "reasoning": "No hallucinations detected. All numbers, dates, and contextual information are verbatim or directly inferable from the authoritative conversation source." }, "answer": "Based on my search, I found your checking account number. From your bank account setup conversation on November 15, 2024, your new checking account number is **4429853327**.\n\nYou'll also need the routing number for your direct deposit setup, which is **123006800**.\n\nJust to confirm - this was for the checking account you opened at the new bank (not your previous Wells Fargo account, which was 8847293001). Is this the account you want to use for your work direct deposit?", "trace": [ { "event": "search_memory", "query": "checking account number direct deposit", "chunk_ids": [ "3e0cdf40421392c0", "3a006babbe4e3fb2", "8ca88f6573db460e", "4e4ffea07db242b6", "577692430b65653c" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrated perfect memory retrieval and contextual understanding. It provided the exact checking account number requested, included the necessary routing number for direct deposit setup, and correctly distinguished between accounts. The proactive confirmation question shows appropriate care for accuracy.", "evaluation_suggestions": "No improvements needed. The response is exemplary for memory agent performance.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "openai-small", "reranker": "none", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 15493.218917000377, "cost_usd": 5.6380000000000006e-05, "input_tokens": 5679, "output_tokens": 521, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.006456, "USD": 5.6380000000000006e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 0.5, "retrieval_mrr": 0.25, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 0.5, "embedding_index_latency_ms": 1726.816417000009, "embedding_index_cost_usd": 5.632e-05, "embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "fixed_query_retrieval_latency_ms": 640.7745839997006, "fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 20, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "4e4ffea07db242b6", "15d7bbeb79e1d7d2", "3e0cdf40421392c0", "8ca88f6573db460e", "814f5b20045c6cb6" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly stated the exact checking account number 4429853327 without any digit errors or confusion with the savings account number 4429853328. The routing number 123006800 was also correctly recalled.", "evidence": [ "assistant: ...your new checking account number is 4429853327", "user: Let me write that down... 4429853327, right?", "assistant: Correct" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent retrieved all necessary information: the checking account number (4429853327) and the routing number (123006800) which is relevant for direct deposit setup as specified in the evaluation criteria. No required information was missing.", "evidence": [ "assistant: Our routing number is 123006800, and your new checking account number is 4429853327" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly distinguished between checking and savings accounts (4429853327 vs 4429853328) and appropriately linked the routing number to the direct deposit context. The mention of TechCorp as the employer was contextually relevant since the user mentioned setting up direct deposit 'at work' and TechCorp was previously established as the employer.", "evidence": [ "user: Yes, my paycheck from TechCorp", "assistant: I can provide you with our routing number and your new account number for that" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent proactively included the routing number 123006800, which is essential for direct deposit setup, even though the user only asked for the account number. This matches the 'excellent' example in the rubric of 'includes the remembered routing number when direct-deposit setup makes it useful.'", "evidence": [ "Agent Response: Your checking account number is 4429853327. You can use this along with the routing number 123006800 to set up direct deposit" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All facts in agent response verified: 4429853327 (checking account), 123006800 (routing number), TechCorp (employer)" ], "reasoning": "No material factual claims were invented or contradicted. The account number, routing number, and employer reference all appear in the authoritative source." }, "answer": "Your checking account number is 4429853327. You can use this along with the routing number 123006800 to set up direct deposit for your paycheck from TechCorp.", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "4e4ffea07db242b6", "15d7bbeb79e1d7d2", "3e0cdf40421392c0", "8ca88f6573db460e", "814f5b20045c6cb6" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrated perfect memory retrieval for this straightforward factual query. It provided the exact checking account number without confusion with the savings account, proactively included the routing number essential for direct deposit, and correctly contextualized with the employer information. This represents ideal performance for a Layer 1 (basic factual recall) test case.", "evaluation_suggestions": "None. The response is exemplary for this test case type.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "openai-small", "reranker": "doubao-semantic", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 29219.73133299798, "cost_usd": 5.6460000000000005e-05, "input_tokens": 7362, "output_tokens": 990, "unpriced_tokens": 0, "cached_input_tokens": 161, "cost_by_currency": { "CNY": 0.015571699999999999, "USD": 5.6460000000000005e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 1726.816417000009, "embedding_index_cost_usd": 5.632e-05, "embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "fixed_query_retrieval_latency_ms": 20426.92737499965, "fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 4.0000000000000003e-07, "CNY": 0.0084904 }, "fixed_query_input_tokens": 3503, "fixed_query_output_tokens": 713, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identifies the exact checking account number 4429853327 without any digit errors. It also correctly provides the routing number 123006800. The agent properly distinguishes between checking (4429853327) and savings (4429853328) accounts, providing only the requested checking account number.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328." ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent retrieves both pieces of information necessary for the user's stated purpose: the checking account number (directly requested) and the routing number (needed for direct deposit setup, as the user explicitly mentioned). All information required by the evaluation criteria is present.", "evidence": [ "your checking account number is 4429853327", "For your direct deposit setup, you'll also need the routing number, which is 123006800", "user: Yes, my paycheck from TechCorp." ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly links the user's mention of 'direct deposit at work' to the prior context where direct deposit was discussed (TechCorp paycheck). It retrieves the account number from the correct temporal point in the conversation and maintains proper distinction between the two account types. No ambiguity or conflict exists in this single-session retrieval.", "evidence": [ "user: Yes, my paycheck from TechCorp.", "assistant: I can provide you with our routing number and your new account number for that.", "agent response: For your direct deposit setup, you'll also need the routing number" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrates excellent proactivity by anticipating and providing the routing number (123006800) that is necessary for direct deposit setup, even though the user only asked for the account number. This directly addresses the user's stated purpose ('to set up my direct deposit at work') with relevant, safe, and useful additional information. The agent also appropriately offers further assistance.", "evidence": [ "For your direct deposit setup, you'll also need the routing number, which is 123006800", "Is there anything else you need for setting up your direct deposit?" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "checking account number is 4429853327", "routing number is 123006800" ], "reasoning": "All factual claims in the agent's response are directly supported by the authoritative conversation source. The account number 4429853327, routing number 123006800, and the context of direct deposit for TechCorp are all verbatim or accurately paraphrased from the source. No invented or contradicted information is present." }, "answer": "Based on your conversation history from when you opened your account at First National Bank, your checking account number is **4429853327**.\n\nFor your direct deposit setup, you'll also need the routing number, which is **123006800**.\n\nIs there anything else you need for setting up your direct deposit?", "trace": [ { "event": "search_memory", "query": "checking account number direct deposit bank account", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent performs excellently across all dimensions. It precisely retrieves the correct checking account number without confusion with the savings account or other numbers (SSN, phone, etc.). It recalls all necessary information including the routing number relevant to the user's direct deposit purpose. It reasons correctly about the context linking 'work' to 'TechCorp' and proactively provides the routing number that completes the information needed for direct deposit setup. No hallucinations are present.", "evaluation_suggestions": "No improvements needed. The response is a model example of accurate memory retrieval with appropriate contextual awareness and proactive assistance.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "openai-small", "reranker": "doubao-semantic", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 34637.54900099957, "cost_usd": 5.6380000000000006e-05, "input_tokens": 8201, "output_tokens": 1194, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.013857600000000001, "USD": 5.6380000000000006e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 1726.816417000009, "embedding_index_cost_usd": 5.632e-05, "embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "fixed_query_retrieval_latency_ms": 20426.92737499965, "fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 4.0000000000000003e-07, "CNY": 0.0084904 }, "fixed_query_input_tokens": 3503, "fixed_query_output_tokens": 713, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent provides the exact checking account number 4429853327 with no errors. It correctly distinguishes from the savings account number 4429853328 and does not confuse with other numbers like SSN (547-82-9163), phone (503-555-8924), or driver's license (D758392).", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328." ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent retrieves both the checking account number and the routing number, which is explicitly called for in the evaluation criteria since the user mentioned direct deposit setup. All information necessary for the question and evaluation criteria is present.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Yes, my paycheck from TechCorp." ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly links the user's question about direct deposit to the prior context where the checking account number was provided for that exact purpose (TechCorp paycheck direct deposit). The temporal connection between the November 15, 2024 setup and current query is correctly established.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Yes, my paycheck from TechCorp." ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent proactively offers the routing number 123006800, recognizing that direct deposit setup requires both numbers. This is exactly the 'useful, safe next-step help' described in the excellent example for proactivity.", "evidence": [ "Let me know if you need the routing number (123006800) as well for the direct deposit form." ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims (checking account number 4429853327, routing number 123006800, date November 15, 2024, TechCorp employer) are directly supported by the authoritative source." ], "reasoning": "No material factual claims are unsupported or contradicted by the source. The agent's response is fully grounded in the conversation transcript." }, "answer": "Your checking account number is 4429853327. This was confirmed during your account setup on November 15, 2024, when you set up direct deposit for your TechCorp paycheck. Let me know if you need the routing number (123006800) as well for the direct deposit form.", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrates perfect memory retrieval for this Layer 1 (basic factual recall) test case. It provides the exact checking account number, correctly distinguishes it from the savings account, links it to the direct deposit context, and proactively offers the routing number. All dimensions score excellently with no hallucinations detected.", "evaluation_suggestions": "No improvements needed. The response is a model example of accurate factual retrieval with appropriate contextual awareness and proactive assistance.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "openai-small", "reranker": "kimi-semantic", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 7102.169083999797, "cost_usd": 5.6380000000000006e-05, "input_tokens": 7693, "output_tokens": 165, "unpriced_tokens": 0, "cached_input_tokens": 161, "cost_by_currency": { "CNY": 0.0224297, "USD": 5.6380000000000006e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 1726.816417000009, "embedding_index_cost_usd": 5.632e-05, "embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "fixed_query_retrieval_latency_ms": 3755.6929999991553, "fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 4.0000000000000003e-07, "CNY": 0.012367 }, "fixed_query_input_tokens": 2928, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830", "3e0cdf40421392c0", "3a006babbe4e3fb2", "4e4ffea07db242b6" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identifies the exact checking account number 4429853327 without any digit errors. The agent also correctly provides the routing number 123006800. There is no confusion with the savings account number 4429853328 or any other numbers in the conversation (SSN 547-82-9163, phone 503-555-8924, Wells Fargo account 8847293001, etc.).", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328." ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent successfully recalls both pieces of information explicitly requested by the evaluation criteria: the checking account number (4429853327) and the routing number (123006800) for direct deposit setup. The agent also correctly contextualizes this as the Premium Checking account opened on November 15, 2024, which is accurate per the conversation timestamp.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "Conversation bank_setup_001 at 2024-11-15 10:30:00", "user: Yes, my paycheck from TechCorp." ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly links the user's question about direct deposit setup to the specific context in the conversation where this information was originally provided (when the user mentioned wanting to set up direct deposit for their TechCorp paycheck). The agent distinguishes between checking and savings accounts appropriately, providing only the checking account number as requested while correctly noting the account type (Premium Checking).", "evidence": [ "user: Yes, my paycheck from TechCorp.", "assistant: I can provide you with our routing number and your new account number for that.", "assistant: You've opened a Premium Checking with account number 4429853327 and a Basic Savings with account number 4429853328." ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrates excellent proactivity by anticipating that the user needs both the checking account number AND the routing number for direct deposit setup, even though the user only explicitly asked for the account number. This directly matches the evaluation criteria's expected behavior. The agent provides both numbers in a helpful format for workplace direct deposit setup without adding irrelevant or unsafe information.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "Agent response: 'This was set up when you opened your Premium Checking account at First National Bank on November 15, 2024. The routing number for your bank is 123006800 — you'll need both for your direct deposit setup at work.'" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims verified: checking account number 4429853327, routing number 123006800, Premium Checking account type, November 15, 2024 date, First National Bank name, direct deposit context for TechCorp paycheck." ], "reasoning": "No material factual claims are absent from or contradicted by the authoritative source. The agent's response contains only information directly supported by the conversation transcript." }, "answer": "Based on your conversation history, your checking account number is **4429853327**.\n\nThis was set up when you opened your Premium Checking account at First National Bank on November 15, 2024. The routing number for your bank is **123006800** — you'll need both for your direct deposit setup at work.", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830", "3e0cdf40421392c0", "3a006babbe4e3fb2", "4e4ffea07db242b6" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent performs excellently across all dimensions. It accurately retrieves the specific checking account number (4429853327) without confusion with the savings account (4429853328) or any other numbers. It proactively includes the routing number (123006800) which, while not explicitly requested, is clearly needed for the stated purpose of setting up direct deposit. All information is precisely correct and fully supported by the source conversation.", "evaluation_suggestions": "No improvements needed. The response is a model example of accurate memory retrieval with appropriate proactive assistance.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "openai-small", "reranker": "kimi-semantic", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 16543.63787499915, "cost_usd": 5.6380000000000006e-05, "input_tokens": 7980, "output_tokens": 459, "unpriced_tokens": 0, "cached_input_tokens": 2892, "cost_by_currency": { "CNY": 0.008148599999999999, "USD": 5.6380000000000006e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 1726.816417000009, "embedding_index_cost_usd": 5.632e-05, "embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "fixed_query_retrieval_latency_ms": 3755.6929999991553, "fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 4.0000000000000003e-07, "CNY": 0.012367 }, "fixed_query_input_tokens": 2928, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830", "3e0cdf40421392c0" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The checking account number 4429853327 is stated exactly as provided in the source. The routing number 123006800 is also correct. No confusion with the savings account number 4429853328 or any other numbers in the conversation.", "evidence": [ "assistant: ...your new checking account number is 4429853327", "user: Let me write that down... 4429853327, right?", "assistant: Correct" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The response provides the exact checking account number requested and includes the routing number, which is directly relevant to the user's stated purpose of setting up direct deposit. Both pieces of information were discussed together in the original conversation for this exact purpose.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identifies that the user needs the checking account number (not savings) and recognizes the contextual relevance of the routing number for direct deposit setup. The temporal link to 'November 15, 2024' is accurate based on the conversation timestamp.", "evidence": [ "user: Yes, my paycheck from TechCorp", "assistant: I can provide you with our routing number and your new account number for that" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent proactively offers the routing number (123006800) unprompted, recognizing it's needed for direct deposit setup. This directly addresses the evaluation criterion that the agent 'should ideally also mention the routing number' given the direct deposit context.", "evidence": [ "Let me know if you need the routing number (123006800) as well for the setup" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims (4429853327, 123006800, November 15, 2024, TechCorp) are directly supported by the authoritative source" ], "reasoning": "No material factual claims are unsupported or contradicted. The date 'November 15, 2024' is correctly derived from the conversation timestamp [2024-11-15 10:30:00]." }, "answer": "Your checking account number is 4429853327. This was confirmed during your account setup on November 15, 2024, when you noted it for direct deposit from TechCorp. Let me know if you need the routing number (123006800) as well for the setup.", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830", "3e0cdf40421392c0" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response demonstrates perfect memory retrieval. It provides the exact checking account number, correctly distinguishes it from the savings account, and proactively includes the routing number which is contextually necessary for the user's stated purpose. All information is traceable to the authoritative source with no errors or unsupported claims.", "evaluation_suggestions": "No improvements needed. The response is a model example of accurate memory retrieval with appropriate contextual awareness.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "qwen3", "reranker": "none", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 25831.329665998965, "cost_usd": 3.078e-05, "input_tokens": 4916, "output_tokens": 69, "unpriced_tokens": 0, "cached_input_tokens": 161, "cost_by_currency": { "CNY": 0.0082697, "USD": 3.078e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 0.5, "retrieval_mrr": 0.3333333333333333, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 4296.359041999494, "embedding_index_cost_usd": 3.07e-05, "embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "fixed_query_retrieval_latency_ms": 17766.350332999536, "fixed_query_retrieval_cost_usd": 2.1e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.1e-07 }, "fixed_query_input_tokens": 21, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "4e4ffea07db242b6", "c466f611e5375254", "8ca88f6573db460e", "3e0cdf40421392c0", "15d7bbeb79e1d7d2" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The checking account number 4429853327 is stated exactly as provided in the source. The routing number 123006800 is also correct. There is no confusion with the savings account number 4429853328 or any other numbers in the conversation (SSN 547-82-9163, phone 503-555-8924, Wells Fargo account 8847293001, etc.).", "evidence": [ "assistant: Our routing number is 123006800, and your new checking account number is 4429853327", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "Both pieces of information necessary for the user's direct deposit setup are retrieved: the checking account number (explicitly requested) and the routing number (contextually necessary). The agent correctly identified that direct deposit setup requires both numbers.", "evidence": [ "assistant: Our routing number is 123006800, and your new checking account number is 4429853327" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly linked the user's question about 'direct deposit at work' to the specific context in the conversation where both the checking account number and routing number were provided together for that exact purpose. No ambiguity or confusion between account types.", "evidence": [ "user: Yes, my paycheck from TechCorp", "assistant: I can provide you with our routing number and your new account number for that" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent proactively included the routing number 123006800, recognizing that direct deposit setup requires both numbers. This is exactly the 'useful, safe next-step help' described in the excellent example for proactivity. The addition is directly relevant and sourced from the same conversation context.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims (checking account 4429853327, routing number 123006800) are directly supported by the authoritative source" ], "reasoning": "No material factual claims are absent from or contradicted by the source. Both numbers appear verbatim in the conversation where the assistant provides them for direct deposit setup." }, "answer": "Based on your conversation history, your checking account number is **4429853327**.\n\nThe bank's routing number is **123006800** — you'll need both of these for your direct deposit setup at work.", "trace": [ { "event": "search_memory", "query": "checking account number bank account direct deposit", "chunk_ids": [ "4e4ffea07db242b6", "c466f611e5375254", "8ca88f6573db460e", "3e0cdf40421392c0", "15d7bbeb79e1d7d2" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response demonstrates perfect memory retrieval for this test case. It correctly identifies the checking account number (distinguishing it from the savings account), provides it accurately, and proactively includes the routing number which is necessary context for the user's stated purpose of setting up direct deposit. All information is grounded in the authoritative source with exact matches.", "evaluation_suggestions": "None. The response is optimal for this retrieval task.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "qwen3", "reranker": "none", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 19531.343333999757, "cost_usd": 3.074e-05, "input_tokens": 5872, "output_tokens": 329, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.0048704, "USD": 3.074e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 0.5, "retrieval_mrr": 0.2, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 4296.359041999494, "embedding_index_cost_usd": 3.07e-05, "embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "fixed_query_retrieval_latency_ms": 17766.350332999536, "fixed_query_retrieval_cost_usd": 2.1e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.1e-07 }, "fixed_query_input_tokens": 21, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "814f5b20045c6cb6", "4e4ffea07db242b6", "15d7bbeb79e1d7d2", "c466f611e5375254", "8ca88f6573db460e" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provided the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguished this from the savings account number 4429853328, providing only the checking account number as requested. No numbers were confused or misstated.", "evidence": [ "Agent response: 'Your checking account number is 4429853327'", "Source: 'your new checking account number is 4429853327' and 'And your savings account number is 4429853328'" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent recalled both the checking account number and the routing number, both of which are necessary for setting up direct deposit as mentioned in the evaluation criteria. The agent correctly identified the context (TechCorp paycheck) and provided the complete information needed for the user's stated purpose.", "evidence": [ "Agent response: 'Your checking account number is 4429853327. You can use this along with the routing number 123006800 to set up direct deposit for your TechCorp paycheck'", "Source: 'Our routing number is 123006800, and your new checking account number is 4429853327' and 'Yes, my paycheck from TechCorp'" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly linked the user's question about direct deposit to the previously discussed context (TechCorp paycheck) and provided the relevant account details. The agent properly distinguished between checking and savings accounts and understood the purpose of the request.", "evidence": [ "Agent response mentions 'to set up direct deposit for your TechCorp paycheck'", "Source: 'Yes, my paycheck from TechCorp' and confirmation of account numbers" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent proactively included the routing number 123006800, which is essential for direct deposit setup, even though the user only asked for the account number. The agent also helpfully reminded the user of the TechCorp employment context. This adds useful, safe next-step help exactly when appropriate.", "evidence": [ "Agent response: 'You can use this along with the routing number 123006800 to set up direct deposit for your TechCorp paycheck'", "Evaluation criteria states: 'the agent should ideally also mention the routing number 123006800 which was discussed in that context'" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims in agent response are supported by source: checking account number 4429853327, routing number 123006800, and TechCorp employment" ], "reasoning": "The agent's response contains no material factual claims that are unsupported or contradicted by the authoritative source. All numbers, names, and context references are verified in the conversation transcript." }, "answer": "Your checking account number is 4429853327. You can use this along with the routing number 123006800 to set up direct deposit for your TechCorp paycheck. Let me know if you need help with anything else!", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "814f5b20045c6cb6", "4e4ffea07db242b6", "15d7bbeb79e1d7d2", "c466f611e5375254", "8ca88f6573db460e" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrated perfect memory retrieval for this straightforward factual query. It provided the exact checking account number, correctly distinguished it from the savings account, proactively included the routing number needed for direct deposit, and maintained proper context awareness by mentioning TechCorp. This represents ideal behavior for a memory agent handling personal financial information retrieval.", "evaluation_suggestions": "No improvements needed. The response is concise, accurate, and appropriately proactive.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "qwen3", "reranker": "doubao-semantic", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 23666.16629299915, "cost_usd": 3.0760000000000004e-05, "input_tokens": 7611, "output_tokens": 691, "unpriced_tokens": 0, "cached_input_tokens": 1067, "cost_by_currency": { "CNY": 0.009985299999999999, "USD": 3.0760000000000004e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 4296.359041999494, "embedding_index_cost_usd": 3.07e-05, "embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "fixed_query_retrieval_latency_ms": 22812.888833000216, "fixed_query_retrieval_cost_usd": 2.1e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.1e-07, "CNY": 0.0090424 }, "fixed_query_input_tokens": 3504, "fixed_query_output_tokens": 782, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provided the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguished this from the savings account number 4429853328, providing only the checking account number as requested. The date 'November 15, 2024' appears in the extracted memory trace header, so this is supported.", "evidence": [ "assistant: ...your new checking account number is 4429853327", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent recalled both the checking account number (4429853327) and the routing number (123006800), which was explicitly requested in the evaluation criteria as ideal for direct deposit setup context. All necessary information for the user's question and the evaluation criteria is present.", "evidence": [ "assistant: Our routing number is 123006800, and your new checking account number is 4429853327", "user: Let me write that down... 4429853327, right?", "assistant: Correct" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identified that the user was asking about direct deposit setup (mentioned in the question: 'I need it to set up my direct deposit at work') and therefore included the routing number alongside the account number. This shows proper contextual reasoning linking the account number to its intended use case.", "evidence": [ "User Question: What was my checking account number again? I need it to set up my direct deposit at work.", "Agent Response: Your checking account number is **4429853327**. ... The routing number for your direct deposit is 123006800." ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent proactively included the routing number (123006800) which is essential for direct deposit setup, exactly as the evaluation criteria described as ideal behavior. This adds useful, safe next-step help without being excessive or irrelevant.", "evidence": [ "Agent Response: The routing number for your direct deposit is 123006800", "Evaluation Criteria: 'the agent should ideally also mention the routing number 123006800 which was discussed in that context'" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims (checking account number 4429853327, routing number 123006800, date November 15, 2024) are supported by the authoritative conversation source" ], "reasoning": "The agent's response contains no material factual claims that are unsupported or contradicted by the authoritative source. The checking account number, routing number, and date all appear in the source conversation." }, "answer": "Your checking account number is **4429853327**. \n\nThis was provided when you set up your account with First National Bank on November 15, 2024. The routing number for your direct deposit is 123006800.", "trace": [ { "event": "search_memory", "query": "checking account number direct deposit", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrates excellent performance across all dimensions. It accurately recalled the specific checking account number without confusion with the savings account number, proactively included the routing number appropriate to the direct deposit context, and made no factual errors or hallucinations.", "evaluation_suggestions": "None - the response is exemplary for this memory retrieval task.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "qwen3", "reranker": "doubao-semantic", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 30343.878748998577, "cost_usd": 3.074e-05, "input_tokens": 8456, "output_tokens": 914, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.0116176, "USD": 3.074e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 4296.359041999494, "embedding_index_cost_usd": 3.07e-05, "embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "fixed_query_retrieval_latency_ms": 22812.888833000216, "fixed_query_retrieval_cost_usd": 2.1e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.1e-07, "CNY": 0.0090424 }, "fixed_query_input_tokens": 3504, "fixed_query_output_tokens": 782, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The checking account number 4429853327 is stated exactly as it appears in the source, with no digit errors or confusion with the savings account number 4429853328. The routing number 123006800 is also correct.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328." ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "All necessary information for the question is present: the checking account number is directly provided, and the routing number is also included (which was requested in the evaluation criteria as ideal for direct deposit context). The agent correctly identifies this was for TechCorp direct deposit setup.", "evidence": [ "Your checking account number is 4429853327. This was confirmed during your account setup on November 15, 2024, when you set up direct deposit for your TechCorp paycheck. Let me know if you need the routing number (123006800) as well for the direct deposit form." ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly links the user's question about direct deposit to the specific context in the conversation where both the checking account number and routing number were discussed. The agent distinguishes between checking and savings accounts appropriately and provides only the requested checking account number while offering the routing number as additional helpful information.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Yes, my paycheck from TechCorp." ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent proactively offers the routing number 123006800, recognizing that direct deposit setup requires both numbers. This is exactly the 'useful, safe next-step help' described in the excellent example for proactivity. The agent also correctly contextualizes when this was discussed (TechCorp direct deposit).", "evidence": [ "Let me know if you need the routing number (123006800) as well for the direct deposit form." ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims (4429853327, 123006800, November 15, 2024, TechCorp) are directly supported by the authoritative source." ], "reasoning": "No material factual claims are unsupported or contradicted. The date 'November 15, 2024' appears in the conversation header [Conversation bank_setup_001 at 2024-11-15 10:30:00]. All numbers and relationships are verbatim from the source." }, "answer": "Your checking account number is 4429853327. This was confirmed during your account setup on November 15, 2024, when you set up direct deposit for your TechCorp paycheck. Let me know if you need the routing number (123006800) as well for the direct deposit form.", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response demonstrates perfect memory retrieval. The checking account number is exact, the routing number is correctly offered proactively for direct deposit purposes, and all contextual details (TechCorp, date) are accurate. The agent correctly distinguishes between checking and savings accounts without confusion.", "evaluation_suggestions": "No improvements needed. The response is a model example of accurate memory retrieval with appropriate proactive assistance.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "qwen3", "reranker": "kimi-semantic", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 18763.832833999913, "cost_usd": 3.0760000000000004e-05, "input_tokens": 7335, "output_tokens": 125, "unpriced_tokens": 0, "cached_input_tokens": 1185, "cost_by_currency": { "CNY": 0.0157505, "USD": 3.0760000000000004e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 4296.359041999494, "embedding_index_cost_usd": 3.07e-05, "embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "fixed_query_retrieval_latency_ms": 3426.3087509998513, "fixed_query_retrieval_cost_usd": 2.1e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.1e-07, "CNY": 0.011522199999999998 }, "fixed_query_input_tokens": 2929, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 256, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830", "3e0cdf40421392c0" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provided the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguished this from the savings account number 4429853328, which was never mentioned in the response. The routing number 123006800 was also provided accurately.", "evidence": [ "Source: 'Our routing number is 123006800, and your new checking account number is 4429853327'", "Source: 'Correct. And your savings account number is 4429853328'", "Agent response: 'Your checking account number is 4429853327' and 'routing number... is 123006800'" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent recalled all necessary information for the user's question. The checking account number was directly provided, and the routing number was also included—which is explicitly noted in the evaluation criteria as ideal since the user mentioned needing it for direct deposit setup. Both pieces of information were stated in the same context in the source conversation.", "evidence": [ "Source: 'I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327'", "User question context: 'I need it to set up my direct deposit at work'", "Agent response includes both account number and routing number" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly linked the user's stated purpose (direct deposit setup) to the relevant information from the conversation. The agent identified that both the account number and routing number would be needed for this purpose, and correctly retrieved both from the same contextual exchange where they were originally provided together. No ambiguity or confusion between account types.", "evidence": [ "Source context: assistant provided both numbers together when user mentioned direct deposit from TechCorp", "Agent correctly inferred routing number would be helpful based on 'direct deposit' context in user question" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrated excellent proactivity by anticipating the user's need for the routing number based on the explicit mention of 'direct deposit at work.' This is exactly the type of 'useful, safe next-step help' described in the criteria. The agent did not invent any information or provide irrelevant advice.", "evidence": [ "User: 'I need it to set up my direct deposit at work'", "Agent: 'You'll also need the routing number for your direct deposit setup, which is 123006800'", "Evaluation criteria: 'the agent should ideally also mention the routing number 123006800'" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All numerical claims (4429853327, 123006800) verified in source", "Date reference 'November 15th' matches conversation timestamp 2024-11-15", "Bank name 'First National Bank' and account type 'Premium Checking' both confirmed in source" ], "reasoning": "No material factual claims were invented or contradicted. The date 'November 15th' is correctly inferred from the conversation timestamp. All numbers, names, and account details are verbatim or directly supported by the authoritative source." }, "answer": "Your checking account number is **4429853327**.\n\nI found this from when you opened your Premium Checking account at First National Bank on November 15th. You'll also need the routing number for your direct deposit setup, which is **123006800**.", "trace": [ { "event": "search_memory", "query": "checking account number direct deposit", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830", "3e0cdf40421392c0" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response is a model example of accurate memory retrieval. It provides the exact checking account number requested, correctly distinguishes it from the savings account, and proactively includes the routing number based on the user's stated need for direct deposit setup. All information is factually grounded in the source conversation with no hallucinations or errors.", "evaluation_suggestions": "No improvements needed. The response demonstrates optimal memory agent behavior for this use case.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "qwen3", "reranker": "kimi-semantic", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 29419.81066599874, "cost_usd": 3.074e-05, "input_tokens": 7882, "output_tokens": 351, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.016363799999999998, "USD": 3.074e-05 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 4296.359041999494, "embedding_index_cost_usd": 3.07e-05, "embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "fixed_query_retrieval_latency_ms": 3426.3087509998513, "fixed_query_retrieval_cost_usd": 2.1e-07, "fixed_query_retrieval_cost_by_currency": { "USD": 2.1e-07, "CNY": 0.011522199999999998 }, "fixed_query_input_tokens": 2929, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 256, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provided the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguished it from the savings account number 4429853328. The routing number 123006800 was also stated correctly. No numbers were confused with other identifiers in the conversation (SSN 547-82-9163, phone 503-555-8924, Wells Fargo account 8847293001, etc.).", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328.", "Agent response: Your checking account number is 4429853327.", "Agent response: routing number (123006800)" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent retrieved both pieces of information necessary for the user's stated purpose: the checking account number for direct deposit setup, and proactively the routing number which is required for direct deposit forms. The context of TechCorp as the employer was also correctly recalled and mentioned.", "evidence": [ "user: Yes, my paycheck from TechCorp.", "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "Agent response: for setting up direct deposit from TechCorp", "Agent response: Let me know if you need the routing number (123006800) as well for the direct deposit form." ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identified that the user needed the checking account number (not savings), linked this to the direct deposit context with TechCorp, and understood that routing numbers are paired with account numbers for direct deposit setup. No ambiguity existed as only one checking account was opened.", "evidence": [ "assistant: You've opened a Premium Checking with account number 4429853327 and a Basic Savings with account number 4429853328.", "Agent response: Your checking account number is 4429853327. This was confirmed during your account setup on November 15, 2024, for setting up direct deposit from TechCorp." ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrated excellent proactivity by anticipating that direct deposit setup requires both the account number and routing number, offering the routing number 123006800 without being explicitly asked. This directly addresses the user's stated need ('I need it to set up my direct deposit at work') with complete actionable information.", "evidence": [ "User question: What was my checking account number again? I need it to set up my direct deposit at work.", "Agent response: Let me know if you need the routing number (123006800) as well for the direct deposit form." ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All numerical claims (4429853327, 123006800, TechCorp) are directly supported by the authoritative source.", "Date reference 'November 15, 2024' matches the conversation timestamp [Conversation bank_setup_001 at 2024-11-15 10:30:00]." ], "reasoning": "No material factual claims are unsupported or contradicted. The account number, routing number, employer name, and date are all verifiable in the authoritative conversation source. The agent's offer to provide the routing number is presented as helpful next-step assistance, not as recalled fact." }, "answer": "Your checking account number is 4429853327. This was confirmed during your account setup on November 15, 2024, for setting up direct deposit from TechCorp. Let me know if you need the routing number (123006800) as well for the direct deposit form.", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response demonstrates perfect memory retrieval and contextual understanding. It provides the exact checking account number requested, correctly distinguishes it from the savings account, recalls the direct deposit context with TechCorp, and proactively offers the routing number needed to complete the direct deposit setup. No errors, omissions, or hallucinations detected.", "evaluation_suggestions": "No improvements needed. The response is exemplary for memory-based retrieval tasks.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "mistral", "reranker": "none", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 4978.6733760010975, "cost_usd": 0.0003489000000000001, "input_tokens": 5455, "output_tokens": 85, "unpriced_tokens": 0, "cached_input_tokens": 673, "cost_by_currency": { "CNY": 0.007428100000000001, "USD": 0.0003489000000000001 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 2159.7906670003795, "embedding_index_cost_usd": 0.00034820000000000006, "embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "fixed_query_retrieval_latency_ms": 690.8573340006114, "fixed_query_retrieval_cost_usd": 2.2e-06, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-06 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "3e0cdf40421392c0", "4e4ffea07db242b6", "15d7bbeb79e1d7d2", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provided the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguished this from the savings account number 4429853328. The routing number 123006800 was also stated accurately. No numbers were confused with other numbers in the conversation (SSN 547-82-9163, phone 503-555-8924, Wells Fargo account 8847293001, etc.).", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328.", "Agent Response: Based on your conversation history, your checking account number is **4429853327**.", "Agent Response: The routing number for your bank is 123006800" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent successfully recalled both the checking account number and the routing number, which together provide complete information needed for direct deposit setup. The evaluation criteria specifically noted that 'the agent should ideally also mention the routing number 123006800 which was discussed in that context.' The agent met this ideal standard by including both pieces of information.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "Agent Response: your checking account number is **4429853327**", "Agent Response: The routing number for your bank is 123006800, which you'll also need for setting up your direct deposit" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identified that the user needed the checking account number specifically (not the savings account), and linked this need to the direct deposit context mentioned in the user's question. The agent appropriately retrieved the information from the correct conversation context where direct deposit was originally discussed. No ambiguity or confusion between the two account numbers.", "evidence": [ "user: Yes, my paycheck from TechCorp.", "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "User Question: What was my checking account number again? I need it to set up my direct deposit at work.", "Agent Response: Based on your conversation history, your checking account number is **4429853327**" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent proactively included the routing number 123006800, recognizing that direct deposit setup requires both the account number and routing number. This was explicitly identified as ideal behavior in the evaluation criteria: 'the agent should ideally also mention the routing number 123006800 which was discussed in that context.' The agent went beyond minimal answer requirements to provide genuinely useful next-step information.", "evidence": [ "Agent Response: The routing number for your bank is 123006800, which you'll also need for setting up your direct deposit", "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327." ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims in the agent response are supported by the authoritative source: checking account number 4429853327, routing number 123006800, bank name First National Bank, date November 15th, and account type Premium Checking" ], "reasoning": "The agent's response contains no material factual claims that are unsupported or contradicted by the authoritative conversation source. The checking account number, routing number, bank name, and date are all directly verifiable from the source conversation." }, "answer": "Based on your conversation history, your checking account number is **4429853327**.\n\nThis was provided when you opened your Premium Checking account with First National Bank on November 15th. The routing number for your bank is 123006800, which you'll also need for setting up your direct deposit.", "trace": [ { "event": "search_memory", "query": "checking account number direct deposit", "chunk_ids": [ "8ca88f6573db460e", "3e0cdf40421392c0", "4e4ffea07db242b6", "15d7bbeb79e1d7d2", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrated excellent performance across all dimensions. It accurately retrieved the specific checking account number requested, correctly distinguished it from the savings account number, proactively included the routing number needed for direct deposit setup, and made no errors in any factual claims. The response fully meets the expected behavior described in the evaluation criteria.", "evaluation_suggestions": "No improvements needed. The response is exemplary for this memory retrieval task.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "mistral", "reranker": "none", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 13415.679042000193, "cost_usd": 0.0003487000000000001, "input_tokens": 6347, "output_tokens": 357, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.005144, "USD": 0.0003487000000000001 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 0.5, "retrieval_mrr": 0.3333333333333333, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 2159.7906670003795, "embedding_index_cost_usd": 0.00034820000000000006, "embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "fixed_query_retrieval_latency_ms": 690.8573340006114, "fixed_query_retrieval_cost_usd": 2.2e-06, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-06 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "15d7bbeb79e1d7d2", "4e4ffea07db242b6", "8ca88f6573db460e", "3e0cdf40421392c0", "814f5b20045c6cb6" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly retrieved the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguished this from the savings account number 4429853328 and did not confuse it with other numbers in the conversation (SSN 547-82-9163, phone 503-555-8924, Wells Fargo account 8847293001, etc.).", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328." ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent recalled both the checking account number (4429853327) and the routing number (123006800) that were specifically discussed in the context of direct deposit setup. This fully satisfies the evaluation criteria which expected both pieces of information given the user's stated purpose of setting up direct deposit at work.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Yes, my paycheck from TechCorp." ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly linked the user's question about direct deposit to the specific context in the conversation where account numbers were provided for that exact purpose (TechCorp paycheck direct deposit). The agent appropriately retrieved both the account number and routing number that were paired together in the original direct deposit discussion.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327.", "user: Yes, my paycheck from TechCorp." ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent proactively included the routing number 123006800 in addition to the checking account number, recognizing that both are needed for direct deposit setup. This matches the expected behavior in the evaluation criteria. The agent also helpfully mentioned the specific context (TechCorp paycheck) showing memory of the employment details, and offered further assistance.", "evidence": [ "Agent Response: Your checking account number is 4429853327. You can use this along with the routing number 123006800 to set up direct deposit for your TechCorp paycheck.", "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327." ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims in the agent response are directly supported by the authoritative source: checking account number 4429853327, routing number 123006800, and TechCorp as the employer for direct deposit." ], "reasoning": "No hallucinations detected. The agent response contains only information explicitly present in the conversation source." }, "answer": "Your checking account number is 4429853327. You can use this along with the routing number 123006800 to set up direct deposit for your TechCorp paycheck. Let me know if you need help with anything else!", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "15d7bbeb79e1d7d2", "4e4ffea07db242b6", "8ca88f6573db460e", "3e0cdf40421392c0", "814f5b20045c6cb6" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrated perfect memory retrieval for this test case. It correctly identified the checking account number (4429853327), distinguished it from the savings account number (4429853328), and proactively included the routing number (123006800) that was originally provided in the same direct deposit context. The agent also correctly remembered the user's employer (TechCorp) and the purpose of the direct deposit. This is an exemplary demonstration of Layer 1 (basic factual recall) memory capabilities.", "evaluation_suggestions": "No improvements needed. The response is complete and accurate.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "mistral", "reranker": "doubao-semantic", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 37011.91766699958, "cost_usd": 0.0003489000000000001, "input_tokens": 8024, "output_tokens": 1287, "unpriced_tokens": 0, "cached_input_tokens": 1067, "cost_by_currency": { "CNY": 0.015052300000000001, "USD": 0.0003489000000000001 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 2159.7906670003795, "embedding_index_cost_usd": 0.00034820000000000006, "embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "fixed_query_retrieval_latency_ms": 28700.594749999254, "fixed_query_retrieval_cost_usd": 2.2e-06, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-06, "CNY": 0.010922399999999999 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 1017, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provided the exact checking account number 4429853327 without any digit errors or confusion with the savings account number 4429853328. The routing number 123006800 was also correctly stated.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327", "user: Let me write that down... 4429853327, right?", "assistant: Correct. And your savings account number is 4429853328" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent retrieved all necessary information: the checking account number (4429853327) directly answers the user's question, and the routing number (123006800) was included as contextually relevant for direct deposit setup, matching the evaluation criteria's expected behavior.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly linked the user's question about direct deposit setup to the specific context in the conversation where both the account number and routing number were discussed together. It appropriately distinguished between the checking account (requested) and savings account (not requested) without confusion.", "evidence": [ "user: Yes, my paycheck from TechCorp", "assistant: I can provide you with our routing number and your new account number for that" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrated excellent proactivity by anticipating the user's need for the routing number 123006800, which is required for direct deposit setup at work. This was explicitly mentioned in the evaluation criteria as ideal behavior. The additional information was directly relevant and safe.", "evidence": [ "The routing number for your bank is 123006800, in case you need that as well for your direct deposit setup at work" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "checking account number 4429853327 matches source", "routing number 123006800 matches source", "context of TechCorp direct deposit matches source" ], "reasoning": "All factual claims in the agent response are directly supported by the authoritative conversation source. The account number, routing number, and context about direct deposit setup are all verbatim or accurately paraphrased from the source. No invented or contradicted information detected." }, "answer": "Based on your conversation history from November 15th, 2024, your checking account number is **4429853327**.\n\nThis was provided when you were setting up direct deposit for your paycheck from TechCorp. The routing number for your bank is 123006800, in case you need that as well for your direct deposit setup at work.", "trace": [ { "event": "search_memory", "query": "checking account number direct deposit", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response demonstrates perfect memory retrieval for this test case. It accurately recalled the checking account number 4429853327, correctly distinguished it from the savings account number, and proactively included the routing number 123006800 which is necessary for the stated purpose of setting up direct deposit. All information is precisely grounded in the authoritative source with no hallucinations or errors.", "evaluation_suggestions": "No improvements needed. The response perfectly meets all evaluation criteria.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "mistral", "reranker": "doubao-semantic", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 35391.53200000055, "cost_usd": 0.0003487000000000001, "input_tokens": 8869, "output_tokens": 1181, "unpriced_tokens": 0, "cached_input_tokens": 0, "cost_by_currency": { "CNY": 0.013753600000000001, "USD": 0.0003487000000000001 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 2159.7906670003795, "embedding_index_cost_usd": 0.00034820000000000006, "embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "fixed_query_retrieval_latency_ms": 28700.594749999254, "fixed_query_retrieval_cost_usd": 2.2e-06, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-06, "CNY": 0.010922399999999999 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 1017, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provides the exact checking account number 4429853327 without any digit errors. The agent also correctly distinguishes this from the savings account number 4429853328, providing only the checking account number as requested. The routing number 123006800 is also stated correctly.", "evidence": [ "Agent response: 'Your checking account number is 4429853327'", "Source: 'Our routing number is 123006800, and your new checking account number is 4429853327'", "Source: 'Correct. And your savings account number is 4429853328'" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent successfully retrieves both pieces of information necessary for the user's direct deposit setup: the checking account number and the routing number. Both were explicitly requested in the evaluation criteria as relevant to the direct deposit context.", "evidence": [ "Agent response: 'Your checking account number is 4429853327. You may also need the routing number 123006800 for setting up direct deposit'", "Source: 'I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327'" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly interprets the user's need (setting up direct deposit at work) and links this to the appropriate information from the conversation. The agent understands that direct deposit requires both account number and routing number, and proactively offers the routing number even though the user only asked for the account number. No ambiguity or conflicts exist in this single-session retrieval.", "evidence": [ "User question: 'What was my checking account number again? I need it to set up my direct deposit at work.'", "Agent response: 'Your checking account number is 4429853327. You may also need the routing number 123006800 for setting up direct deposit—let me know if you need that as well!'", "Source context: 'Yes, my paycheck from TechCorp.' followed by provision of both numbers for direct deposit setup" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrates excellent proactivity by anticipating that direct deposit setup requires both the account number and routing number, and offers the routing number unprompted. This is exactly the behavior described in the evaluation criteria as expected. The agent also invites further assistance with 'let me know if you need that as well!'", "evidence": [ "Agent response: 'You may also need the routing number 123006800 for setting up direct deposit—let me know if you need that as well!'", "Evaluation criteria: 'Since the user is asking about setting up direct deposit at work, the agent should ideally also mention the routing number 123006800 which was discussed in that context'" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All factual claims in agent response verified against source:", "Checking account number 4429853327: confirmed in source", "Routing number 123006800: confirmed in source", "Context of direct deposit setup: confirmed in source with TechCorp paycheck discussion" ], "reasoning": "No hallucinations detected. All numerical values and contextual claims are directly supported by the authoritative conversation source. The agent's inference that routing number would be needed for direct deposit is logical reasoning, not a factual claim about memory." }, "answer": "Your checking account number is 4429853327. You may also need the routing number 123006800 for setting up direct deposit—let me know if you need that as well!", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response is exemplary for this memory retrieval task. It provides the exact checking account number requested, correctly distinguishes it from the savings account number, and proactively offers the routing number in the context of direct deposit setup—precisely matching the expected behavior described in the evaluation criteria. No errors, omissions, or hallucinations are present.", "evaluation_suggestions": "No improvements needed. The response perfectly meets all evaluation criteria.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "mistral", "reranker": "kimi-semantic", "main_model": "kimi", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 6380.75104300151, "cost_usd": 0.0003489000000000001, "input_tokens": 7450, "output_tokens": 107, "unpriced_tokens": 0, "cached_input_tokens": 1067, "cost_by_currency": { "CNY": 0.0145699, "USD": 0.0003489000000000001 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 2159.7906670003795, "embedding_index_cost_usd": 0.00034820000000000006, "embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "fixed_query_retrieval_latency_ms": 3269.574332999582, "fixed_query_retrieval_cost_usd": 2.2e-06, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-06, "CNY": 0.0106774 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 512, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provides the exact checking account number 4429853327 without any digit errors. It also correctly distinguishes this from the savings account number 4429853328, providing only the requested checking account number. The routing number 123006800 is also stated correctly.", "evidence": [ "your checking account number is 4429853327", "The routing number is 123006800", "Source: 'Our routing number is 123006800, and your new checking account number is 4429853327'", "Source: 'Correct. And your savings account number is 4429853328'" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent retrieves all necessary information: the checking account number itself, and proactively includes the routing number which is essential for direct deposit setup as mentioned in the evaluation criteria. The agent also correctly recalls the context (TechCorp direct deposit) from the original conversation.", "evidence": [ "checking account number is 4429853327", "routing number is 123006800", "setting up direct deposit for your paycheck from TechCorp", "Source: 'Yes, my paycheck from TechCorp' and 'Our routing number is 123006800, and your new checking account number is 4429853327'" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly identifies that the user is asking about direct deposit setup (implied by 'set up my direct deposit at work') and links this to the prior context where routing number was provided alongside the account number. The agent appropriately distinguishes between checking and savings accounts without confusion.", "evidence": [ "You had mentioned setting up direct deposit for your paycheck from TechCorp at that time", "checking account number is 4429853327 (not savings account 4429853328)", "Source context shows account numbers were provided together during direct deposit discussion" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent demonstrates excellent proactivity by including the routing number 123006800, which is essential for direct deposit setup and was explicitly mentioned in the evaluation criteria as desirable. The agent also usefully reminds the user of the prior TechCorp direct deposit context, reinforcing the relevance of the information provided.", "evidence": [ "The routing number is 123006800", "You had mentioned setting up direct deposit for your paycheck from TechCorp at that time", "Evaluation criteria: 'the agent should ideally also mention the routing number 123006800 which was discussed in that context'" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "All facts verified against source: checking account 4429853327, routing number 123006800, TechCorp employment, direct deposit context" ], "reasoning": "No material factual claims are unsupported or contradicted by the authoritative source. All numbers, names, and context references are directly traceable to the conversation transcript." }, "answer": "Based on your conversation from November 15th, your checking account number is **4429853327**. The routing number is 123006800. You had mentioned setting up direct deposit for your paycheck from TechCorp at that time.", "trace": [ { "event": "search_memory", "query": "checking account number direct deposit", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent response is exemplary. It provides the exact checking account number requested, correctly distinguishes it from the savings account number, proactively includes the routing number essential for direct deposit, and maintains proper context linkage to the original conversation. All information is factually accurate per the authoritative source.", "evaluation_suggestions": "None. The response fully meets all evaluation criteria.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null }, { "experiment": "7-11", "test_id": "layer1_01_bank_account", "layer": "layer1", "system": "rag", "embedding": "mistral", "reranker": "kimi-semantic", "main_model": "doubao", "success": true, "reward": 1.0, "steps": 2, "tool_calls": 1, "latency_ms": 17369.281625999974, "cost_usd": 0.0003487000000000001, "input_tokens": 9339, "output_tokens": 474, "unpriced_tokens": 0, "cached_input_tokens": 256, "cost_by_currency": { "CNY": 0.0178842, "USD": 0.0003487000000000001 }, "unpriced_requests": 0, "retrieval_hit_at_5": 1.0, "retrieval_recall_at_5": 1.0, "retrieval_mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "embedding_index_latency_ms": 2159.7906670003795, "embedding_index_cost_usd": 0.00034820000000000006, "embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "fixed_query_retrieval_latency_ms": 3269.574332999582, "fixed_query_retrieval_cost_usd": 2.2e-06, "fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-06, "CNY": 0.0106774 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 512, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "retrieved_chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830", "3e0cdf40421392c0", "15d7bbeb79e1d7d2", "4e4ffea07db242b6" ], "rubric_dimensions": { "precision": 4, "recall": 4, "reasoning": 4, "proactivity": 4 }, "rubric_details": { "precision": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly provided the exact checking account number 4429853327 without any digit errors. The agent also correctly identified the routing number as 123006800. There was no confusion with the savings account number (4429853328) or any other numbers in the conversation.", "evidence": [ "Your checking account number is 4429853327", "routing number as well (it's 123006800)" ], "boundary_case": null }, "recall": { "grade": "excellent", "score": 4, "reasoning": "The agent recalled both the checking account number and the routing number, which together provide complete information needed for direct deposit setup. The user explicitly asked for the checking account number for direct deposit purposes, and the agent remembered both pieces of information that were originally provided in that exact context.", "evidence": [ "assistant: I can provide you with our routing number and your new account number for that. Our routing number is 123006800, and your new checking account number is 4429853327", "user: Let me write that down... 4429853327, right? assistant: Correct" ], "boundary_case": null }, "reasoning": { "grade": "excellent", "score": 4, "reasoning": "The agent correctly linked the user's question about direct deposit setup to the specific context where both the account number and routing number were originally discussed. The agent appropriately distinguished between checking and savings accounts and understood that routing numbers are relevant for direct deposit setup.", "evidence": [ "You can use this for setting up your direct deposit at work. Let me know if you need the routing number as well" ], "boundary_case": null }, "proactivity": { "grade": "excellent", "score": 4, "reasoning": "The agent proactively offered the routing number (123006800) in addition to the checking account number, recognizing that direct deposit setup requires both pieces of information. This demonstrates helpful anticipation of the user's complete needs based on the stated purpose (direct deposit at work).", "evidence": [ "Let me know if you need the routing number as well (it's 123006800)" ], "boundary_case": null } }, "hallucination_veto": false, "hallucination_detail": { "detected": false, "claims": [], "evidence": [ "Account number 4429853327 confirmed in source: 'your new checking account number is 4429853327'", "Routing number 123006800 confirmed in source: 'Our routing number is 123006800'" ], "reasoning": "All factual claims in the agent response are directly supported by the authoritative conversation source. The checking account number, routing number, and the context of direct deposit setup are all verbatim or accurately paraphrased from the source material." }, "answer": "Your checking account number is 4429853327. You can use this for setting up your direct deposit at work. Let me know if you need the routing number as well (it's 123006800).", "trace": [ { "event": "search_memory", "query": "checking account number", "chunk_ids": [ "8ca88f6573db460e", "74834bc228dbe830", "3e0cdf40421392c0", "15d7bbeb79e1d7d2", "4e4ffea07db242b6" ] }, { "event": "answer" } ], "evaluation_reasoning": "The agent demonstrated perfect memory retrieval for this straightforward factual query. It provided the exact checking account number requested, correctly distinguished it from the savings account number, and proactively included the routing number which is necessary for direct deposit setup. The response is concise, accurate, and helpful.", "evaluation_suggestions": "No improvements needed. The response is exemplary for this type of factual retrieval task.", "cost_accounting": {}, "evidence_mode": "real_api", "status": "ok", "error": null } ], "summary": [ { "layer": "layer1", "embedding": "bge-m3", "reranker": "doubao-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 25902.4385849998, "total_cost_usd": 2.9690000000000002e-05, "total_cost_by_currency": { "CNY": 0.011433600000000002, "USD": 2.9690000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 16083.018499999525, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0078184, "USD": 2.2e-07 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 629, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "bge-m3", "reranker": "doubao-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 0.916667, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 33213.47191800032, "total_cost_usd": 2.9710000000000002e-05, "total_cost_by_currency": { "CNY": 0.015945099999999997, "USD": 2.9710000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 161, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 16083.018499999525, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0078184, "USD": 2.2e-07 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 629, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "bge-m3", "reranker": "kimi-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 16903.2389590011, "total_cost_usd": 2.9690000000000002e-05, "total_cost_by_currency": { "CNY": 0.0173958, "USD": 2.9690000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 3120.466540999587, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.012367, "USD": 2.2e-07 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "bge-m3", "reranker": "kimi-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 8581.173708001188, "total_cost_usd": 2.9710000000000002e-05, "total_cost_by_currency": { "CNY": 0.0195353, "USD": 2.9710000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 929, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 3120.466540999587, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.012367, "USD": 2.2e-07 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "bge-m3", "reranker": "none", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 15984.19020900019, "total_cost_usd": 2.9690000000000002e-05, "total_cost_by_currency": { "CNY": 0.006056000000000001, "USD": 2.9690000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.5, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 396.5964580002037, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-07 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "bge-m3", "reranker": "none", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 4872.473999999784, "total_cost_usd": 2.9710000000000002e-05, "total_cost_by_currency": { "CNY": 0.009522999999999998, "USD": 2.9710000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 0.5, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 396.5964580002037, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-07 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "mistral", "reranker": "doubao-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 35391.53200000055, "total_cost_usd": 0.0003487000000000001, "total_cost_by_currency": { "CNY": 0.013753600000000001, "USD": 0.0003487000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 28700.594749999254, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.010922399999999999, "USD": 2.2e-06 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 1017, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "mistral", "reranker": "doubao-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 37011.91766699958, "total_cost_usd": 0.0003489000000000001, "total_cost_by_currency": { "CNY": 0.015052300000000001, "USD": 0.0003489000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 1067, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 28700.594749999254, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.010922399999999999, "USD": 2.2e-06 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 1017, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "mistral", "reranker": "kimi-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 17369.281625999974, "total_cost_usd": 0.0003487000000000001, "total_cost_by_currency": { "CNY": 0.0178842, "USD": 0.0003487000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 256, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 3269.574332999582, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0106774, "USD": 2.2e-06 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 512, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "mistral", "reranker": "kimi-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 6380.75104300151, "total_cost_usd": 0.0003489000000000001, "total_cost_by_currency": { "CNY": 0.0145699, "USD": 0.0003489000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 1067, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 3269.574332999582, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0106774, "USD": 2.2e-06 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 512, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "mistral", "reranker": "none", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 13415.679042000193, "total_cost_usd": 0.0003487000000000001, "total_cost_by_currency": { "CNY": 0.005144, "USD": 0.0003487000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.3333333333333333, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 690.8573340006114, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-06 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "mistral", "reranker": "none", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 4978.6733760010975, "total_cost_usd": 0.0003489000000000001, "total_cost_by_currency": { "CNY": 0.007428100000000001, "USD": 0.0003489000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 673, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 690.8573340006114, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-06 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "openai-small", "reranker": "doubao-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 34637.54900099957, "total_cost_usd": 5.6380000000000006e-05, "total_cost_by_currency": { "CNY": 0.013857600000000001, "USD": 5.6380000000000006e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 20426.92737499965, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0084904, "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 3503, "fixed_query_output_tokens": 713, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "openai-small", "reranker": "doubao-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 29219.73133299798, "total_cost_usd": 5.6460000000000005e-05, "total_cost_by_currency": { "CNY": 0.015571699999999999, "USD": 5.6460000000000005e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 161, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 20426.92737499965, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0084904, "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 3503, "fixed_query_output_tokens": 713, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "openai-small", "reranker": "kimi-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 16543.63787499915, "total_cost_usd": 5.6380000000000006e-05, "total_cost_by_currency": { "CNY": 0.008148599999999999, "USD": 5.6380000000000006e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 2892, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 3755.6929999991553, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.012367, "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 2928, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "openai-small", "reranker": "kimi-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 7102.169083999797, "total_cost_usd": 5.6380000000000006e-05, "total_cost_by_currency": { "CNY": 0.0224297, "USD": 5.6380000000000006e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 161, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 3755.6929999991553, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.012367, "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 2928, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "openai-small", "reranker": "none", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 15493.218917000377, "total_cost_usd": 5.6380000000000006e-05, "total_cost_by_currency": { "CNY": 0.006456, "USD": 5.6380000000000006e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.25, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 0.5, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 640.7745839997006, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 20, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "openai-small", "reranker": "none", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 5347.5308330007465, "total_cost_usd": 5.6420000000000005e-05, "total_cost_by_currency": { "CNY": 0.008840899999999999, "USD": 5.6420000000000005e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 417, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.3333333333333333, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 0.5, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 640.7745839997006, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 20, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "qwen3", "reranker": "doubao-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 30343.878748998577, "total_cost_usd": 3.074e-05, "total_cost_by_currency": { "CNY": 0.0116176, "USD": 3.074e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 22812.888833000216, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0090424, "USD": 2.1e-07 }, "fixed_query_input_tokens": 3504, "fixed_query_output_tokens": 782, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "qwen3", "reranker": "doubao-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 23666.16629299915, "total_cost_usd": 3.0760000000000004e-05, "total_cost_by_currency": { "CNY": 0.009985299999999999, "USD": 3.0760000000000004e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 1067, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 22812.888833000216, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0090424, "USD": 2.1e-07 }, "fixed_query_input_tokens": 3504, "fixed_query_output_tokens": 782, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "qwen3", "reranker": "kimi-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 29419.81066599874, "total_cost_usd": 3.074e-05, "total_cost_by_currency": { "CNY": 0.016363799999999998, "USD": 3.074e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 3426.3087509998513, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.011522199999999998, "USD": 2.1e-07 }, "fixed_query_input_tokens": 2929, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 256, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "qwen3", "reranker": "kimi-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 18763.832833999913, "total_cost_usd": 3.0760000000000004e-05, "total_cost_by_currency": { "CNY": 0.0157505, "USD": 3.0760000000000004e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 1185, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 3426.3087509998513, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.011522199999999998, "USD": 2.1e-07 }, "fixed_query_input_tokens": 2929, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 256, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "qwen3", "reranker": "none", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 19531.343333999757, "total_cost_usd": 3.074e-05, "total_cost_by_currency": { "CNY": 0.0048704, "USD": 3.074e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.2, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 17766.350332999536, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.1e-07 }, "fixed_query_input_tokens": 21, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "layer": "layer1", "embedding": "qwen3", "reranker": "none", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 25831.329665998965, "total_cost_usd": 3.078e-05, "total_cost_by_currency": { "CNY": 0.0082697, "USD": 3.078e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 161, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.3333333333333333, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 17766.350332999536, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.1e-07 }, "fixed_query_input_tokens": 21, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 } ], "configuration_errors": [], "cost_note": "Costs are reported by the provider's published native currency using dated, source-linked pricing. cost_usd is retained only for USD-denominated charges; currencies are never silently converted. Both unpriced tokens and unpriced requests must be zero for completion.", "scope_note": "latency and cost cover memory ingestion/retrieval/reranking/main-Agent calls; benchmark gold selection and LLM judging are evaluation overhead and excluded", "interaction_analysis": { "analysis_scope": { "expected_case_count": 60, "expected_cells_per_case": 24, "expected_trajectory_count": 1440, "observed_case_count": 1, "configured_trajectory_count": 24, "successful_trajectory_count": 24, "error_trajectory_count": 0, "selection_conclusions_allowed": false, "scope_status": "partial_descriptive_only" }, "factorial_cells": [ { "embedding": "bge-m3", "reranker": "doubao-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 25902.4385849998, "total_cost_usd": 2.9690000000000002e-05, "total_cost_by_currency": { "CNY": 0.011433600000000002, "USD": 2.9690000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 16083.018499999525, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0078184, "USD": 2.2e-07 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 629, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "bge-m3", "reranker": "doubao-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 0.916667, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 33213.47191800032, "total_cost_usd": 2.9710000000000002e-05, "total_cost_by_currency": { "CNY": 0.015945099999999997, "USD": 2.9710000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 161, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 16083.018499999525, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0078184, "USD": 2.2e-07 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 629, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "bge-m3", "reranker": "kimi-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 16903.2389590011, "total_cost_usd": 2.9690000000000002e-05, "total_cost_by_currency": { "CNY": 0.0173958, "USD": 2.9690000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 3120.466540999587, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.012367, "USD": 2.2e-07 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "bge-m3", "reranker": "kimi-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 8581.173708001188, "total_cost_usd": 2.9710000000000002e-05, "total_cost_by_currency": { "CNY": 0.0195353, "USD": 2.9710000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 929, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 3120.466540999587, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.012367, "USD": 2.2e-07 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "bge-m3", "reranker": "none", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 15984.19020900019, "total_cost_usd": 2.9690000000000002e-05, "total_cost_by_currency": { "CNY": 0.006056000000000001, "USD": 2.9690000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.5, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 396.5964580002037, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-07 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "bge-m3", "reranker": "none", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 4872.473999999784, "total_cost_usd": 2.9710000000000002e-05, "total_cost_by_currency": { "CNY": 0.009522999999999998, "USD": 2.9710000000000002e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 0.5, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 2.964e-05, "total_embedding_index_cost_by_currency": { "USD": 2.964e-05 }, "average_fixed_query_retrieval_latency_ms": 396.5964580002037, "total_fixed_query_retrieval_cost_usd": 2.2e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-07 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "mistral", "reranker": "doubao-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 35391.53200000055, "total_cost_usd": 0.0003487000000000001, "total_cost_by_currency": { "CNY": 0.013753600000000001, "USD": 0.0003487000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 28700.594749999254, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.010922399999999999, "USD": 2.2e-06 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 1017, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "mistral", "reranker": "doubao-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 37011.91766699958, "total_cost_usd": 0.0003489000000000001, "total_cost_by_currency": { "CNY": 0.015052300000000001, "USD": 0.0003489000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 1067, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 28700.594749999254, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.010922399999999999, "USD": 2.2e-06 }, "fixed_query_input_tokens": 3505, "fixed_query_output_tokens": 1017, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "mistral", "reranker": "kimi-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 17369.281625999974, "total_cost_usd": 0.0003487000000000001, "total_cost_by_currency": { "CNY": 0.0178842, "USD": 0.0003487000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 256, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 3269.574332999582, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0106774, "USD": 2.2e-06 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 512, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "mistral", "reranker": "kimi-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 6380.75104300151, "total_cost_usd": 0.0003489000000000001, "total_cost_by_currency": { "CNY": 0.0145699, "USD": 0.0003489000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 1067, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 3269.574332999582, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0106774, "USD": 2.2e-06 }, "fixed_query_input_tokens": 2930, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 512, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "mistral", "reranker": "none", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 13415.679042000193, "total_cost_usd": 0.0003487000000000001, "total_cost_by_currency": { "CNY": 0.005144, "USD": 0.0003487000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.3333333333333333, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 690.8573340006114, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-06 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "mistral", "reranker": "none", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 4978.6733760010975, "total_cost_usd": 0.0003489000000000001, "total_cost_by_currency": { "CNY": 0.007428100000000001, "USD": 0.0003489000000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 673, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.00034820000000000006, "total_embedding_index_cost_by_currency": { "USD": 0.00034820000000000006 }, "average_fixed_query_retrieval_latency_ms": 690.8573340006114, "total_fixed_query_retrieval_cost_usd": 2.2e-06, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.2e-06 }, "fixed_query_input_tokens": 22, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "openai-small", "reranker": "doubao-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 34637.54900099957, "total_cost_usd": 5.6380000000000006e-05, "total_cost_by_currency": { "CNY": 0.013857600000000001, "USD": 5.6380000000000006e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 20426.92737499965, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0084904, "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 3503, "fixed_query_output_tokens": 713, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "openai-small", "reranker": "doubao-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 29219.73133299798, "total_cost_usd": 5.6460000000000005e-05, "total_cost_by_currency": { "CNY": 0.015571699999999999, "USD": 5.6460000000000005e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 161, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 20426.92737499965, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0084904, "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 3503, "fixed_query_output_tokens": 713, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "openai-small", "reranker": "kimi-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 16543.63787499915, "total_cost_usd": 5.6380000000000006e-05, "total_cost_by_currency": { "CNY": 0.008148599999999999, "USD": 5.6380000000000006e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 2892, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 3755.6929999991553, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.012367, "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 2928, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "openai-small", "reranker": "kimi-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 7102.169083999797, "total_cost_usd": 5.6380000000000006e-05, "total_cost_by_currency": { "CNY": 0.0224297, "USD": 5.6380000000000006e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 161, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 3755.6929999991553, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.012367, "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 2928, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "openai-small", "reranker": "none", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 15493.218917000377, "total_cost_usd": 5.6380000000000006e-05, "total_cost_by_currency": { "CNY": 0.006456, "USD": 5.6380000000000006e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.25, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 0.5, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 640.7745839997006, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 20, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "openai-small", "reranker": "none", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 5347.5308330007465, "total_cost_usd": 5.6420000000000005e-05, "total_cost_by_currency": { "CNY": 0.008840899999999999, "USD": 5.6420000000000005e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 417, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.3333333333333333, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 0.5, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 5.632e-05, "total_embedding_index_cost_by_currency": { "USD": 5.632e-05 }, "average_fixed_query_retrieval_latency_ms": 640.7745839997006, "total_fixed_query_retrieval_cost_usd": 4.0000000000000003e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 4.0000000000000003e-07 }, "fixed_query_input_tokens": 20, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "qwen3", "reranker": "doubao-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 30343.878748998577, "total_cost_usd": 3.074e-05, "total_cost_by_currency": { "CNY": 0.0116176, "USD": 3.074e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 22812.888833000216, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0090424, "USD": 2.1e-07 }, "fixed_query_input_tokens": 3504, "fixed_query_output_tokens": 782, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "qwen3", "reranker": "doubao-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 23666.16629299915, "total_cost_usd": 3.0760000000000004e-05, "total_cost_by_currency": { "CNY": 0.009985299999999999, "USD": 3.0760000000000004e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 1067, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 22812.888833000216, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0090424, "USD": 2.1e-07 }, "fixed_query_input_tokens": 3504, "fixed_query_output_tokens": 782, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "qwen3", "reranker": "kimi-semantic", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 29419.81066599874, "total_cost_usd": 3.074e-05, "total_cost_by_currency": { "CNY": 0.016363799999999998, "USD": 3.074e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 3426.3087509998513, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.011522199999999998, "USD": 2.1e-07 }, "fixed_query_input_tokens": 2929, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 256, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "qwen3", "reranker": "kimi-semantic", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 18763.832833999913, "total_cost_usd": 3.0760000000000004e-05, "total_cost_by_currency": { "CNY": 0.0157505, "USD": 3.0760000000000004e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 1185, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 3426.3087509998513, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.011522199999999998, "USD": 2.1e-07 }, "fixed_query_input_tokens": 2929, "fixed_query_output_tokens": 35, "fixed_query_cached_input_tokens": 256, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "qwen3", "reranker": "none", "main_model": "doubao", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 19531.343333999757, "total_cost_usd": 3.074e-05, "total_cost_by_currency": { "CNY": 0.0048704, "USD": 3.074e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 0, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.2, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 17766.350332999536, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.1e-07 }, "fixed_query_input_tokens": 21, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "qwen3", "reranker": "none", "main_model": "kimi", "configured_cases": 1, "completed_cases": 1, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 25831.329665998965, "total_cost_usd": 3.078e-05, "total_cost_by_currency": { "CNY": 0.0082697, "USD": 3.078e-05 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 161, "hit_at_5": 1.0, "recall_at_5": 0.5, "mrr": 0.3333333333333333, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 3.07e-05, "total_embedding_index_cost_by_currency": { "USD": 3.07e-05 }, "average_fixed_query_retrieval_latency_ms": 17766.350332999536, "total_fixed_query_retrieval_cost_usd": 2.1e-07, "total_fixed_query_retrieval_cost_by_currency": { "USD": 2.1e-07 }, "fixed_query_input_tokens": 21, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 } ], "embedding_marginals": [ { "embedding": "bge-m3", "configured_cases": 6, "completed_cases": 6, "error_cases": 0, "success_rate": 1.0, "average_reward": 0.9861111666666668, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 17576.164563167065, "total_cost_usd": 0.00017820000000000002, "total_cost_by_currency": { "CNY": 0.0798888, "USD": 0.00017820000000000002 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 1090, "hit_at_5": 1.0, "recall_at_5": 0.9166666666666666, "mrr": 0.8333333333333334, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.8333333333333334, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 1986.3613750003424, "total_embedding_index_cost_usd": 0.00017784, "total_embedding_index_cost_by_currency": { "USD": 0.00017784 }, "average_fixed_query_retrieval_latency_ms": 6533.360499666439, "total_fixed_query_retrieval_cost_usd": 1.32e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0403708, "USD": 1.32e-06 }, "fixed_query_input_tokens": 12914, "fixed_query_output_tokens": 1328, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "mistral", "configured_cases": 6, "completed_cases": 6, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 19091.305792333816, "total_cost_usd": 0.0020928, "total_cost_by_currency": { "CNY": 0.0738321, "USD": 0.0020928 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 3063, "hit_at_5": 1.0, "recall_at_5": 0.9166666666666666, "mrr": 0.8888888888888888, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.8333333333333334, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2159.7906670003795, "total_embedding_index_cost_usd": 0.0020892000000000003, "total_embedding_index_cost_by_currency": { "USD": 0.0020892000000000003 }, "average_fixed_query_retrieval_latency_ms": 10887.008805666483, "total_fixed_query_retrieval_cost_usd": 1.3199999999999999e-05, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.043199600000000005, "USD": 1.3199999999999999e-05 }, "fixed_query_input_tokens": 12914, "fixed_query_output_tokens": 2104, "fixed_query_cached_input_tokens": 1024, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "openai-small", "configured_cases": 6, "completed_cases": 6, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 18057.306173832938, "total_cost_usd": 0.00033840000000000004, "total_cost_by_currency": { "CNY": 0.0753045, "USD": 0.00033840000000000004 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 3631, "hit_at_5": 1.0, "recall_at_5": 0.8333333333333334, "mrr": 0.7638888888888888, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.8333333333333334, "fixed_query_mrr": 0.8333333333333334, "average_embedding_index_latency_ms": 1726.816417000009, "total_embedding_index_cost_usd": 0.00033792, "total_embedding_index_cost_by_currency": { "USD": 0.00033792 }, "average_fixed_query_retrieval_latency_ms": 8274.464986332836, "total_fixed_query_retrieval_cost_usd": 2.4000000000000003e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.041714799999999996, "USD": 2.4000000000000003e-06 }, "fixed_query_input_tokens": 12902, "fixed_query_output_tokens": 1496, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "embedding": "qwen3", "configured_cases": 6, "completed_cases": 6, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 24592.72692366585, "total_cost_usd": 0.00018452, "total_cost_by_currency": { "CNY": 0.0668573, "USD": 0.00018452 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 2413, "hit_at_5": 1.0, "recall_at_5": 0.8333333333333334, "mrr": 0.7555555555555555, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.8333333333333334, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 4296.359041999494, "total_embedding_index_cost_usd": 0.00018419999999999998, "total_embedding_index_cost_by_currency": { "USD": 0.00018419999999999998 }, "average_fixed_query_retrieval_latency_ms": 14668.515972333202, "total_fixed_query_retrieval_cost_usd": 1.2600000000000002e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0411292, "USD": 1.2600000000000002e-06 }, "fixed_query_input_tokens": 12908, "fixed_query_output_tokens": 1634, "fixed_query_cached_input_tokens": 512, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 } ], "reranker_marginals": [ { "reranker": "doubao-semantic", "configured_cases": 8, "completed_cases": 8, "error_cases": 0, "success_rate": 1.0, "average_reward": 0.989583375, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 31173.335693249443, "total_cost_usd": 0.0009313400000000002, "total_cost_by_currency": { "CNY": 0.10721680000000001, "USD": 0.0009313400000000002 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 2456, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2542.3318752500563, "total_embedding_index_cost_usd": 0.0009297200000000002, "total_embedding_index_cost_by_currency": { "USD": 0.0009297200000000002 }, "average_fixed_query_retrieval_latency_ms": 22005.85736449966, "total_fixed_query_retrieval_cost_usd": 6.0600000000000004e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0725472, "USD": 6.0600000000000004e-06 }, "fixed_query_input_tokens": 28034, "fixed_query_output_tokens": 6282, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "reranker": "kimi-semantic", "configured_cases": 8, "completed_cases": 8, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 15132.986974375171, "total_cost_usd": 0.0009312600000000002, "total_cost_by_currency": { "CNY": 0.1320778, "USD": 0.0009312600000000002 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 6490, "hit_at_5": 1.0, "recall_at_5": 1.0, "mrr": 1.0, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 1.0, "fixed_query_mrr": 1.0, "average_embedding_index_latency_ms": 2542.3318752500563, "total_embedding_index_cost_usd": 0.0009297200000000002, "total_embedding_index_cost_by_currency": { "USD": 0.0009297200000000002 }, "average_fixed_query_retrieval_latency_ms": 3393.010656249544, "total_fixed_query_retrieval_cost_usd": 6.0600000000000004e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.0938672, "USD": 6.0600000000000004e-06 }, "fixed_query_input_tokens": 23434, "fixed_query_output_tokens": 280, "fixed_query_cached_input_tokens": 1536, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "reranker": "none", "configured_cases": 8, "completed_cases": 8, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 13181.804922125139, "total_cost_usd": 0.0009313200000000001, "total_cost_by_currency": { "CNY": 0.0565881, "USD": 0.0009313200000000001 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 1251, "hit_at_5": 1.0, "recall_at_5": 0.625, "mrr": 0.43125, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.5, "fixed_query_mrr": 0.875, "average_embedding_index_latency_ms": 2542.3318752500563, "total_embedding_index_cost_usd": 0.0009297200000000002, "total_embedding_index_cost_by_currency": { "USD": 0.0009297200000000002 }, "average_fixed_query_retrieval_latency_ms": 4873.644677250013, "total_fixed_query_retrieval_cost_usd": 6.0600000000000004e-06, "total_fixed_query_retrieval_cost_by_currency": { "USD": 6.0600000000000004e-06 }, "fixed_query_input_tokens": 170, "fixed_query_output_tokens": 0, "fixed_query_cached_input_tokens": 0, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 } ], "main_model_marginals": [ { "main_model": "doubao", "configured_cases": 12, "completed_cases": 12, "error_cases": 0, "success_rate": 1.0, "average_reward": 1.0, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 22577.983246916498, "total_cost_usd": 0.0013965300000000004, "total_cost_by_currency": { "CNY": 0.1329812, "USD": 0.0013965300000000004 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 3148, "hit_at_5": 1.0, "recall_at_5": 0.8333333333333334, "mrr": 0.7736111111111111, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.8333333333333334, "fixed_query_mrr": 0.9583333333333334, "average_embedding_index_latency_ms": 2542.3318752500563, "total_embedding_index_cost_usd": 0.0013945800000000003, "total_embedding_index_cost_by_currency": { "USD": 0.0013945800000000003 }, "average_fixed_query_retrieval_latency_ms": 10090.83756599974, "total_fixed_query_retrieval_cost_usd": 9.090000000000001e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.08320720000000001, "USD": 9.090000000000001e-06 }, "fixed_query_input_tokens": 25819, "fixed_query_output_tokens": 3281, "fixed_query_cached_input_tokens": 768, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 }, { "main_model": "kimi", "configured_cases": 12, "completed_cases": 12, "error_cases": 0, "success_rate": 1.0, "average_reward": 0.9930555833333333, "average_steps": 2.0, "average_tool_calls": 1.0, "average_latency_ms": 17080.768479583337, "total_cost_usd": 0.0013973900000000003, "total_cost_by_currency": { "CNY": 0.16290149999999998, "USD": 0.0013973900000000003 }, "unpriced_tokens": 0, "unpriced_requests": 0, "cached_input_tokens": 7049, "hit_at_5": 1.0, "recall_at_5": 0.9166666666666666, "mrr": 0.8472222222222222, "fixed_query_hit_at_5": 1.0, "fixed_query_recall_at_5": 0.8333333333333334, "fixed_query_mrr": 0.9583333333333334, "average_embedding_index_latency_ms": 2542.3318752500563, "total_embedding_index_cost_usd": 0.0013945800000000003, "total_embedding_index_cost_by_currency": { "USD": 0.0013945800000000003 }, "average_fixed_query_retrieval_latency_ms": 10090.83756599974, "total_fixed_query_retrieval_cost_usd": 9.090000000000001e-06, "total_fixed_query_retrieval_cost_by_currency": { "CNY": 0.08320720000000001, "USD": 9.090000000000001e-06 }, "fixed_query_input_tokens": 25819, "fixed_query_output_tokens": 3281, "fixed_query_cached_input_tokens": 768, "fixed_query_unpriced_tokens": 0, "fixed_query_unpriced_requests": 0, "hallucination_veto_rate": 0.0 } ], "reranker_value_by_embedding_and_main_model": [ { "embedding": "bge-m3", "main_model": "doubao", "reranker": "doubao-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 9918.24837599961, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.0053776000000000015, "USD": 0.0 } }, { "embedding": "bge-m3", "main_model": "kimi", "reranker": "doubao-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 28340.99791800054, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.006422099999999998, "USD": 0.0 } }, { "embedding": "bge-m3", "main_model": "doubao", "reranker": "kimi-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 919.0487500009112, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.011339799999999999, "USD": 0.0 } }, { "embedding": "bge-m3", "main_model": "kimi", "reranker": "kimi-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 3708.6997080014044, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.0100123, "USD": 0.0 } }, { "embedding": "mistral", "main_model": "doubao", "reranker": "doubao-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 21975.85295800036, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.008609600000000002, "USD": 0.0 } }, { "embedding": "mistral", "main_model": "kimi", "reranker": "doubao-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 32033.244290998482, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.0076242, "USD": 0.0 } }, { "embedding": "mistral", "main_model": "doubao", "reranker": "kimi-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 3953.602583999782, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.0127402, "USD": 0.0 } }, { "embedding": "mistral", "main_model": "kimi", "reranker": "kimi-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 1402.0776670004125, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.007141799999999999, "USD": 0.0 } }, { "embedding": "openai-small", "main_model": "doubao", "reranker": "doubao-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 19144.330083999193, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.007401600000000001, "USD": 0.0 } }, { "embedding": "openai-small", "main_model": "kimi", "reranker": "doubao-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 23872.200499997234, "cost_usd_delta": 3.999999999999962e-08, "cost_delta_by_currency": { "CNY": 0.0067308, "USD": 3.999999999999962e-08 } }, { "embedding": "openai-small", "main_model": "doubao", "reranker": "kimi-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 1050.4189579987724, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.001692599999999999, "USD": 0.0 } }, { "embedding": "openai-small", "main_model": "kimi", "reranker": "kimi-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 1754.6382509990508, "cost_usd_delta": -3.999999999999962e-08, "cost_delta_by_currency": { "CNY": 0.013588800000000002, "USD": -3.999999999999962e-08 } }, { "embedding": "qwen3", "main_model": "doubao", "reranker": "doubao-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 10812.53541499882, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.0067472, "USD": 0.0 } }, { "embedding": "qwen3", "main_model": "kimi", "reranker": "doubao-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": -2165.163372999814, "cost_usd_delta": -1.999999999999642e-08, "cost_delta_by_currency": { "CNY": 0.0017155999999999994, "USD": -1.999999999999642e-08 } }, { "embedding": "qwen3", "main_model": "doubao", "reranker": "kimi-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": 9888.467331998982, "cost_usd_delta": 0.0, "cost_delta_by_currency": { "CNY": 0.011493399999999997, "USD": 0.0 } }, { "embedding": "qwen3", "main_model": "kimi", "reranker": "kimi-semantic", "success_rate_delta": 0.0, "fixed_query_recall_at_5_delta": 0.5, "latency_ms_delta": -7067.496831999051, "cost_usd_delta": -1.999999999999642e-08, "cost_delta_by_currency": { "CNY": 0.007480800000000001, "USD": -1.999999999999642e-08 } } ], "reranker_redundancy_by_embedding": [ { "embedding": "bge-m3", "mean_reranker_success_delta": 0.0, "mean_reranker_fixed_query_recall_delta": 0.5, "mean_reranker_latency_delta_ms": 10721.748688000616, "reranker_is_redundant_on_observed_cases": false }, { "embedding": "mistral", "mean_reranker_success_delta": 0.0, "mean_reranker_fixed_query_recall_delta": 0.5, "mean_reranker_latency_delta_ms": 14841.194374999759, "reranker_is_redundant_on_observed_cases": false }, { "embedding": "openai-small", "mean_reranker_success_delta": 0.0, "mean_reranker_fixed_query_recall_delta": 0.5, "mean_reranker_latency_delta_ms": 11455.396948248563, "reranker_is_redundant_on_observed_cases": false }, { "embedding": "qwen3", "mean_reranker_success_delta": 0.0, "mean_reranker_fixed_query_recall_delta": 0.5, "mean_reranker_latency_delta_ms": 2867.085635499734, "reranker_is_redundant_on_observed_cases": false } ], "main_model_compensation_when_retrieval_incomplete": [ { "main_model": "doubao", "cases_with_incomplete_retrieval": 4, "success_rate_despite_incomplete_retrieval": 1.0, "average_retrieval_recall_at_5": 0.5 }, { "main_model": "kimi", "cases_with_incomplete_retrieval": 2, "success_rate_despite_incomplete_retrieval": 1.0, "average_retrieval_recall_at_5": 0.5 } ], "interpretation_note": "Compare factorial cells and conditional deltas: a reranker's value is conditional on both embedding and main model. Do not infer a system choice from marginal rankings alone. The official completion gate has not passed; all values are partial diagnostics and must not be used to select a system." } }