liqiang
b119135836
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
380 lines
8.9 KiB
JSON
380 lines
8.9 KiB
JSON
{
|
|
"schema_version": "chapter3-evidence-v1",
|
|
"experiment": "3-5",
|
|
"run_id": "20260729T183232Z-3_5-f9f70b37",
|
|
"provenance": {
|
|
"captured_at": "2026-07-29T18:32:32.564511+00:00",
|
|
"git_revision": "4a7f37cf278bd15948c409f14533017c4c7fbc29",
|
|
"python": "3.11.4 (main, Jul 5 2023, 08:40:20) [Clang 14.0.6 ]",
|
|
"platform": "macOS-26.3-arm64-arm-64bit",
|
|
"credential_presence": {
|
|
"ARK_API_KEY": true,
|
|
"MOONSHOT_API_KEY": true,
|
|
"OPENAI_API_KEY": true,
|
|
"GEMINI_API_KEY": true,
|
|
"SILICONFLOW_API_KEY": true
|
|
}
|
|
},
|
|
"status": "passed",
|
|
"method": {
|
|
"implementation": "chapter3/sparse-embedding/bm25_engine.py",
|
|
"algorithm": "from-scratch inverted index + Robertson/Sparck Jones BM25",
|
|
"third_party_retrieval_library": null
|
|
},
|
|
"hand_calculation": {
|
|
"corpus": [
|
|
"rare rare common",
|
|
"common common",
|
|
"common filler filler filler",
|
|
"filler"
|
|
],
|
|
"term": "rare",
|
|
"doc_id": 0,
|
|
"parameters": {
|
|
"N": 4,
|
|
"df": 1,
|
|
"tf": 2,
|
|
"dl": 3,
|
|
"avgdl": 2.5,
|
|
"k1": 1.5,
|
|
"b": 0.75
|
|
},
|
|
"formula": "ln((N-df+0.5)/(df+0.5)) * tf*(k1+1) / (tf+k1*(1-b+b*dl/avgdl))",
|
|
"intermediate": {
|
|
"independent_raw_idf": 0.8472978603872037,
|
|
"engine_raw_idf": 0.8472978603872037,
|
|
"scoring_idf": 0.8472978603872037,
|
|
"numerator": 5.0,
|
|
"denominator": 3.7249999999999996
|
|
},
|
|
"expected_score": 1.1373125642781259,
|
|
"engine_score": 1.1373125642781259,
|
|
"absolute_error": 0.0,
|
|
"tolerance": 1e-12,
|
|
"posting_list": [
|
|
0
|
|
],
|
|
"recorded_document_frequency": 1,
|
|
"passed": true
|
|
},
|
|
"benchmark": {
|
|
"corpus": [
|
|
{
|
|
"doc_id": "doc_1",
|
|
"title": "Python Language",
|
|
"text": "Python is a high-level programming language known for readability and a simple syntax."
|
|
},
|
|
{
|
|
"doc_id": "doc_2",
|
|
"title": "JavaScript Runtime",
|
|
"text": "JavaScript runs in the browser and on servers via Node.js for full-stack web development."
|
|
},
|
|
{
|
|
"doc_id": "doc_3",
|
|
"title": "Model Distillation",
|
|
"text": "Model distillation compresses a large teacher model into a smaller student model while preserving accuracy."
|
|
},
|
|
{
|
|
"doc_id": "doc_4",
|
|
"title": "Knowledge Distillation",
|
|
"text": "Knowledge distillation transfers knowledge from a big neural network to a compact model for efficient inference."
|
|
},
|
|
{
|
|
"doc_id": "doc_5",
|
|
"title": "BM25 Ranking",
|
|
"text": "BM25 is a probabilistic ranking function using term frequency and inverse document frequency."
|
|
},
|
|
{
|
|
"doc_id": "doc_6",
|
|
"title": "HTTP Errors",
|
|
"text": "The HTTP 404 error code means the requested resource was not found on the web server."
|
|
},
|
|
{
|
|
"doc_id": "doc_7",
|
|
"title": "A Playful Kitten",
|
|
"text": "A cute kitten chased a ball of yarn across the living room floor all afternoon."
|
|
},
|
|
{
|
|
"doc_id": "doc_8",
|
|
"title": "Silent Hunter",
|
|
"text": "The feline predator stalked its prey silently through the tall grass at dusk."
|
|
},
|
|
{
|
|
"doc_id": "doc_9",
|
|
"title": "Hardware Fault",
|
|
"text": "Error code XK9-2B4-7Q1 indicates a hardware fault in the storage controller board."
|
|
},
|
|
{
|
|
"doc_id": "doc_10",
|
|
"title": "Transformers",
|
|
"text": "Transformer models use self-attention to process input sequences in parallel efficiently."
|
|
}
|
|
],
|
|
"labels": {
|
|
"model distillation": [
|
|
"doc_3",
|
|
"doc_4"
|
|
],
|
|
"HTTP 404 error": [
|
|
"doc_6"
|
|
],
|
|
"XK9-2B4-7Q1": [
|
|
"doc_9"
|
|
],
|
|
"BM25 ranking function": [
|
|
"doc_5"
|
|
],
|
|
"cat": [
|
|
"doc_7",
|
|
"doc_8"
|
|
]
|
|
},
|
|
"parameters": {
|
|
"k1": 1.5,
|
|
"b": 0.75,
|
|
"top_k": 5
|
|
},
|
|
"index_statistics": {
|
|
"total_documents": 10,
|
|
"unique_terms": 98,
|
|
"total_terms": 114,
|
|
"average_document_length": 11.4,
|
|
"terms_by_frequency": [
|
|
[
|
|
"model",
|
|
4
|
|
],
|
|
[
|
|
"for",
|
|
3
|
|
],
|
|
[
|
|
"and",
|
|
3
|
|
],
|
|
[
|
|
"in",
|
|
3
|
|
],
|
|
[
|
|
"web",
|
|
2
|
|
],
|
|
[
|
|
"distillation",
|
|
2
|
|
],
|
|
[
|
|
"knowledge",
|
|
2
|
|
],
|
|
[
|
|
"to",
|
|
2
|
|
],
|
|
[
|
|
"frequency",
|
|
2
|
|
],
|
|
[
|
|
"error",
|
|
2
|
|
]
|
|
]
|
|
},
|
|
"build_latency_ms": 1.668,
|
|
"queries": [
|
|
{
|
|
"query": "model distillation",
|
|
"category": "exact-keyword",
|
|
"relevant": [
|
|
"doc_3",
|
|
"doc_4"
|
|
],
|
|
"retrieved": [
|
|
"doc_3",
|
|
"doc_4"
|
|
],
|
|
"hits": [
|
|
"doc_3",
|
|
"doc_4"
|
|
],
|
|
"recall_at_5": 1.0,
|
|
"reciprocal_rank": 1.0,
|
|
"latency_ms": 0.276,
|
|
"results": [
|
|
{
|
|
"rank": 1,
|
|
"doc_id": "doc_3",
|
|
"score": 3.121561765506991,
|
|
"matched_terms": [
|
|
"model",
|
|
"distillation"
|
|
],
|
|
"term_frequencies": {
|
|
"model": 3,
|
|
"distillation": 1
|
|
}
|
|
},
|
|
{
|
|
"rank": 2,
|
|
"doc_id": "doc_4",
|
|
"score": 2.219735866426749,
|
|
"matched_terms": [
|
|
"model",
|
|
"distillation"
|
|
],
|
|
"term_frequencies": {
|
|
"model": 1,
|
|
"distillation": 1
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"query": "HTTP 404 error",
|
|
"category": "exact-keyword",
|
|
"relevant": [
|
|
"doc_6"
|
|
],
|
|
"retrieved": [
|
|
"doc_6",
|
|
"doc_9"
|
|
],
|
|
"hits": [
|
|
"doc_6"
|
|
],
|
|
"recall_at_5": 1.0,
|
|
"reciprocal_rank": 1.0,
|
|
"latency_ms": 0.269,
|
|
"results": [
|
|
{
|
|
"rank": 1,
|
|
"doc_id": "doc_6",
|
|
"score": 4.994285959345282,
|
|
"matched_terms": [
|
|
"HTTP",
|
|
"404",
|
|
"error"
|
|
],
|
|
"term_frequencies": {
|
|
"HTTP": 1,
|
|
"404": 1,
|
|
"error": 1
|
|
}
|
|
},
|
|
{
|
|
"rank": 2,
|
|
"doc_id": "doc_9",
|
|
"score": 1.2953611811041896,
|
|
"matched_terms": [
|
|
"error"
|
|
],
|
|
"term_frequencies": {
|
|
"HTTP": 0,
|
|
"404": 0,
|
|
"error": 1
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"query": "XK9-2B4-7Q1",
|
|
"category": "exact-keyword",
|
|
"relevant": [
|
|
"doc_9"
|
|
],
|
|
"retrieved": [
|
|
"doc_9"
|
|
],
|
|
"hits": [
|
|
"doc_9"
|
|
],
|
|
"recall_at_5": 1.0,
|
|
"reciprocal_rank": 1.0,
|
|
"latency_ms": 0.129,
|
|
"results": [
|
|
{
|
|
"rank": 1,
|
|
"doc_id": "doc_9",
|
|
"score": 1.9537998395246958,
|
|
"matched_terms": [
|
|
"XK9-2B4-7Q1"
|
|
],
|
|
"term_frequencies": {
|
|
"XK9-2B4-7Q1": 1
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"query": "BM25 ranking function",
|
|
"category": "exact-keyword",
|
|
"relevant": [
|
|
"doc_5"
|
|
],
|
|
"retrieved": [
|
|
"doc_5"
|
|
],
|
|
"hits": [
|
|
"doc_5"
|
|
],
|
|
"recall_at_5": 1.0,
|
|
"reciprocal_rank": 1.0,
|
|
"latency_ms": 0.261,
|
|
"results": [
|
|
{
|
|
"rank": 1,
|
|
"doc_id": "doc_5",
|
|
"score": 5.626316650182078,
|
|
"matched_terms": [
|
|
"BM25",
|
|
"ranking",
|
|
"function"
|
|
],
|
|
"term_frequencies": {
|
|
"BM25": 1,
|
|
"ranking": 1,
|
|
"function": 1
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"query": "cat",
|
|
"category": "synonym-only",
|
|
"relevant": [
|
|
"doc_7",
|
|
"doc_8"
|
|
],
|
|
"retrieved": [],
|
|
"hits": [],
|
|
"recall_at_5": 0.0,
|
|
"reciprocal_rank": 0.0,
|
|
"latency_ms": 0.095,
|
|
"results": []
|
|
}
|
|
],
|
|
"metrics": {
|
|
"exact_keyword_recall_at_5": 1.0,
|
|
"exact_keyword_mrr": 1.0,
|
|
"synonym_only_recall_at_5": 0.0,
|
|
"synonym_only_mrr": 0.0
|
|
}
|
|
},
|
|
"summary": {
|
|
"exact_keyword_recall_at_5": 1.0,
|
|
"exact_keyword_mrr": 1.0,
|
|
"synonym_only_recall_at_5": 0.0,
|
|
"synonym_only_mrr": 0.0
|
|
},
|
|
"acceptance": {
|
|
"uses_from_scratch_engine": true,
|
|
"hand_score_matches": true,
|
|
"inverted_index_df_matches": true,
|
|
"all_exact_keyword_queries_recalled": true,
|
|
"synonym_only_failure_observed": true,
|
|
"transparent_tf_idf_scores_retained": true
|
|
}
|
|
}
|