ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter8/exact-copy-sft/validation/tokenizer_audit.json

@@ -0,0 +1,65 @@
+{
+  "probe_count": 512,
+  "tokenizers": {
+    "Qwen/Qwen3-8B": {
+      "vocab_size": 151669,
+      "roundtrip_rate": 0.80078125,
+      "mean_tokens": 34.603515625,
+      "failures": [
+        "eval-004",
+        "eval-009",
+        "eval-014",
+        "eval-019",
+        "eval-024",
+        "eval-029",
+        "eval-034",
+        "eval-039",
+        "eval-044",
+        "eval-049",
+        "eval-054",
+        "eval-059",
+        "eval-064",
+        "eval-069",
+        "eval-074",
+        "eval-079",
+        "eval-084",
+        "eval-089",
+        "eval-094",
+        "eval-099"
+      ]
+    },
+    "Qwen/Qwen2.5-0.5B-Instruct": {
+      "vocab_size": 151665,
+      "roundtrip_rate": 0.80078125,
+      "mean_tokens": 34.603515625,
+      "failures": [
+        "eval-004",
+        "eval-009",
+        "eval-014",
+        "eval-019",
+        "eval-024",
+        "eval-029",
+        "eval-034",
+        "eval-039",
+        "eval-044",
+        "eval-049",
+        "eval-054",
+        "eval-059",
+        "eval-064",
+        "eval-069",
+        "eval-074",
+        "eval-079",
+        "eval-084",
+        "eval-089",
+        "eval-094",
+        "eval-099"
+      ]
+    },
+    "mistralai/Mistral-7B-v0.1": {
+      "vocab_size": 32000,
+      "roundtrip_rate": 1.0,
+      "mean_tokens": 39.26171875,
+      "failures": []
+    }
+  }
+}