ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter8/MultilingualReasoning/evaluate_multilingual.py

@@ -0,0 +1,587 @@
+"""Multilingual Reasoning Evaluator for AI Agent Book (Chapter 7).
+
+Evaluates LLM reasoning models across multiple languages (English, Spanish,
+French, Chinese, Japanese) measuring CoT language fidelity, task accuracy,
+token usage, and cross-lingual transfer efficiency.
+"""
+
+from __future__ import annotations
+
+import inspect
+import math
+import re
+import statistics
+import warnings
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+
+LANG_MAP: dict[str, str] = {
+    "en": "English",
+    "english": "English",
+    "es": "Spanish",
+    "spanish": "Spanish",
+    "fr": "French",
+    "french": "French",
+    "zh": "Chinese",
+    "chinese": "Chinese",
+    "zh-cn": "Chinese",
+    "zh-tw": "Chinese",
+    "ja": "Japanese",
+    "japanese": "Japanese",
+    "de": "German",
+    "german": "German",
+    "it": "Italian",
+    "italian": "Italian",
+    "pt": "Portuguese",
+    "portuguese": "Portuguese",
+    "ru": "Russian",
+    "russian": "Russian",
+    "ko": "Korean",
+    "korean": "Korean",
+    "ar": "Arabic",
+    "arabic": "Arabic",
+    "hi": "Hindi",
+    "hindi": "Hindi",
+    "nl": "Dutch",
+    "dutch": "Dutch",
+    "tr": "Turkish",
+    "turkish": "Turkish",
+}
+
+# Regex patterns for script detection
+CJK_RE = re.compile(r"[\u4e00-\u9fff\u3400-\u4dbf]")
+HIRAGANA_RE = re.compile(r"[\u3040-\u309f]")
+KATAKANA_RE = re.compile(r"[\u30a0-\u30ff]")
+SPANISH_SPECIAL_RE = re.compile(r"[áéíóúüñÁÉÍÓÚÜÑ¿¡]")
+FRENCH_SPECIAL_RE = re.compile(r"[éèêëàâùûçôîïÉÈÊËÀÂÙÛÇÔÎÏ]")
+
+SPANISH_WORDS = {
+    "el", "la", "los", "las", "un", "una", "de", "en", "que", "es", "por", "para",
+    "con", "del", "al", "como", "más", "mas", "pero", "sus", "porque", "entonces",
+    "paso", "respuesta", "solucion", "solución", "por lo tanto", "primero", "luego"
+}
+
+FRENCH_WORDS = {
+    "le", "la", "les", "un", "une", "des", "du", "de", "en", "et", "est", "que",
+    "qui", "pour", "dans", "ce", "sur", "avec", "plus", "par", "mais", "donc",
+    "parce que", "alors", "etape", "étape", "reponse", "réponse", "solution", "premièrement"
+}
+
+ENGLISH_WORDS = {
+    "the", "be", "to", "of", "and", "a", "in", "that", "have", "it", "for",
+    "not", "on", "with", "he", "as", "you", "do", "at", "this", "but", "his",
+    "by", "from", "they", "we", "say", "her", "she", "or", "an", "will", "my",
+    "one", "all", "would", "there", "their", "what", "so", "up", "out", "if",
+    "about", "who", "get", "which", "go", "me", "when", "make", "can", "like",
+    "time", "no", "just", "him", "know", "take", "people", "into", "year",
+    "your", "good", "some", "could", "them", "see", "other", "than", "then",
+    "now", "look", "only", "come", "its", "over", "think", "also", "back",
+    "after", "use", "two", "how", "our", "work", "first", "well", "way",
+    "even", "new", "want", "because", "any", "these", "give", "day", "most",
+    "us", "therefore", "step", "reasoning", "solution", "answer", "equals",
+    "is", "plus", "minus", "times", "divided", "equal", "result"
+}
+
+
+def normalize_language(lang: str) -> str:
+    """Normalize language identifier string to canonical English name."""
+    cleaned = str(lang).strip().lower()
+    canonical = LANG_MAP.get(cleaned)
+    if canonical is not None:
+        return canonical
+    # Unknown language: title-case for consistent multi-word names and warn
+    # so callers notice silent filtering in evaluate().
+    title_cased = cleaned.title()
+    warnings.warn(
+        f"Unrecognized language identifier {lang!r}; normalized to {title_cased!r}. "
+        f"Add it to LANG_MAP for reliable matching.",
+        stacklevel=2,
+    )
+    return title_cased
+
+
+def estimate_tokens(text: str) -> int:
+    """Estimate token count for a string if exact count is unavailable."""
+    if not text:
+        return 0
+    cjk_count = len(CJK_RE.findall(text)) + len(HIRAGANA_RE.findall(text)) + len(KATAKANA_RE.findall(text))
+    non_cjk_text = CJK_RE.sub(" ", HIRAGANA_RE.sub(" ", KATAKANA_RE.sub(" ", text)))
+    words = non_cjk_text.split()
+    # ~1.3 tokens per word for Latin scripts, ~1.5 tokens per character for CJK/Kana
+    return max(1, int(len(words) * 1.3 + cjk_count * 1.5))
+def _extract_token_counts(tu: Any) -> Optional[dict[str, Optional[int]]]:
+    if tu is None:
+        return None
+
+    def get_val(key1: str, key2: Optional[str] = None) -> Optional[int]:
+        val = None
+        if isinstance(tu, dict):
+            val = tu.get(key1)
+            if val is None and key2:
+                val = tu.get(key2)
+        else:
+            val = getattr(tu, key1, None)
+            if val is None and key2:
+                val = getattr(tu, key2, None)
+        if val is not None:
+            try:
+                return int(val)
+            except (ValueError, TypeError):
+                return None
+        return None
+
+    p_tok = get_val("prompt_tokens", "input_tokens")
+    c_tok = get_val("completion_tokens", "output_tokens")
+    r_tok = get_val("reasoning_tokens")
+    t_tok = get_val("total_tokens")
+
+    if any(x is not None for x in (p_tok, c_tok, r_tok, t_tok)):
+        return {
+            "prompt_tokens": p_tok,
+            "completion_tokens": c_tok,
+            "reasoning_tokens": r_tok,
+            "total_tokens": t_tok,
+        }
+    return None
+
+
+def _accepts_language(fn: Any) -> bool:
+    try:
+        sig = inspect.signature(fn)
+        for param in sig.parameters.values():
+            if param.name == "language" or param.kind == inspect.Parameter.VAR_KEYWORD:
+                return True
+        return False
+    except (ValueError, TypeError):
+        return False
+
+
+class MultilingualReasoningEvaluator:
+    """Evaluates reasoning LLMs across multiple languages.
+
+    Computes:
+    - CoT Language Fidelity Score: Consistency of reasoning steps with target language.
+    - Task Accuracy: Correctness of final generated answers.
+    - Token Usage: Breakdown of prompt, completion, and reasoning token costs.
+    - Cross-Lingual Transfer Efficiency: Relative performance across non-English languages.
+    """
+
+    def __init__(self, target_languages: Optional[Sequence[str]] = None) -> None:
+        if target_languages is None:
+            self.target_languages = ["English", "Spanish", "French", "Chinese", "Japanese"]
+        else:
+            self.target_languages = [normalize_language(lang) for lang in target_languages]
+
+    def evaluate_cot_fidelity(self, cot_text: str, target_language: str) -> float:
+        """Calculate language fidelity score (0.0 - 1.0) for Chain-of-Thought text."""
+        if not cot_text or not cot_text.strip():
+            return 0.0
+
+        lang = normalize_language(target_language)
+        text = cot_text.strip()
+        non_space_chars = len(re.sub(r"\s+", "", text))
+        if non_space_chars == 0:
+            return 0.0
+
+        cjk_count = len(CJK_RE.findall(text))
+        hiragana_count = len(HIRAGANA_RE.findall(text))
+        katakana_count = len(KATAKANA_RE.findall(text))
+        kana_count = hiragana_count + katakana_count
+
+        if lang == "Chinese":
+            if kana_count > 0:
+                cjk_ratio = cjk_count / non_space_chars
+                return max(0.0, min(1.0, cjk_ratio / 0.3) * 0.7)
+            cjk_ratio = cjk_count / non_space_chars
+            return min(1.0, cjk_ratio / 0.35)
+
+        if lang == "Japanese":
+            if kana_count > 0:
+                j_ratio = (kana_count + cjk_count) / non_space_chars
+                return min(1.0, j_ratio / 0.35)
+            if cjk_count > 0:
+                return 0.4
+            return 0.0
+
+        # For European / Latin languages, CJK/Kana script implies cross-lingual leakage
+        if cjk_count > 0 or kana_count > 0:
+            return 0.0
+
+        words = [w.lower() for w in re.findall(r"\b[a-zA-ZáéíóúüñÁÉÍÓÚÜÑàâäèêëîïôöùûüçÀÂÄÉÈÊËÎÏÔÖÙÛÜÇ]+\b", text)]
+        total_words = len(words)
+        if total_words == 0:
+            return 1.0 if any(c.isalnum() for c in text) else 0.0
+
+        if lang == "Spanish":
+            special_count = len(SPANISH_SPECIAL_RE.findall(text))
+            spanish_word_matches = sum(1 for w in words if w in SPANISH_WORDS)
+            score = (special_count * 2 + spanish_word_matches) / max(1, total_words)
+            return min(1.0, max(0.2, score * 3.0))
+
+        if lang == "French":
+            special_count = len(FRENCH_SPECIAL_RE.findall(text))
+            french_word_matches = sum(1 for w in words if w in FRENCH_WORDS)
+            score = (special_count * 2 + french_word_matches) / max(1, total_words)
+            return min(1.0, max(0.2, score * 3.0))
+
+        if lang == "English":
+            english_word_matches = sum(1 for w in words if w in ENGLISH_WORDS)
+            french_special = len(FRENCH_SPECIAL_RE.findall(text))
+            spanish_special = len(SPANISH_SPECIAL_RE.findall(text))
+            penalty = (french_special + spanish_special) * 0.1
+            match_ratio = english_word_matches / max(1, total_words)
+            score = match_ratio * 1.8 - penalty
+            return min(1.0, max(0.0, score))
+        return 0.5
+
+    def evaluate_accuracy(self, predicted_answer: str, reference_answer: str) -> float:
+        """Determine task accuracy (1.0 for match, 0.0 for mismatch)."""
+        pred = str(predicted_answer if predicted_answer is not None else "").strip().lower()
+        ref = str(reference_answer if reference_answer is not None else "").strip().lower()
+
+        if not pred or not ref:
+            return 0.0
+
+        if pred == ref:
+            return 1.0
+
+        # Strip common trailing punctuation
+        pred_clean = re.sub(r"[.,;!\?]+$", "", pred)
+        ref_clean = re.sub(r"[.,;!\?]+$", "", ref)
+        if pred_clean == ref_clean:
+            return 1.0
+
+        # Try numeric comparison
+        pred_nums = re.findall(r"[-+]?\d*\.?\d+", pred)
+        ref_nums = re.findall(r"[-+]?\d*\.?\d+", ref)
+        if pred_nums and ref_nums:
+            try:
+                p_val = float(pred_nums[-1])
+                r_val = float(ref_nums[-1])
+                if math.isclose(p_val, r_val, rel_tol=1e-4, abs_tol=1e-4):
+                    return 1.0
+            except ValueError:
+                pass
+
+        # Substring matching for references with word boundaries
+        target_ref = ref_clean if ref_clean else ref
+        if target_ref:
+            # \b word boundaries don't work for CJK characters; use direct
+            # containment for non-ASCII references, and \b for Latin text.
+            if re.search(r"[^\x00-\x7f]", target_ref):
+                if target_ref in pred_clean or target_ref in pred:
+                    return 1.0
+            else:
+                pattern = r"\b" + re.escape(target_ref) + r"\b"
+                if re.search(pattern, pred_clean) or re.search(pattern, pred):
+                    return 1.0
+        return 0.0
+
+    def compute_token_usage(
+        self, prompt: str, reasoning: str, answer: str, model_output: Any = None
+    ) -> dict[str, int]:
+        """Extract or estimate prompt, completion, reasoning, and total token usage."""
+        tu = None
+        if isinstance(model_output, dict):
+            if "token_usage" in model_output and model_output["token_usage"] is not None:
+                tu = model_output["token_usage"]
+            else:
+                tu = model_output
+        elif model_output is not None:
+            if hasattr(model_output, "token_usage") and getattr(model_output, "token_usage") is not None:
+                tu = getattr(model_output, "token_usage")
+            else:
+                tu = model_output
+
+        extracted = _extract_token_counts(tu)
+        if extracted is not None:
+            r_tok = extracted["reasoning_tokens"] if extracted["reasoning_tokens"] is not None else estimate_tokens(reasoning)
+            p_tok = extracted["prompt_tokens"] if extracted["prompt_tokens"] is not None else estimate_tokens(prompt)
+            c_tok = extracted["completion_tokens"] if extracted["completion_tokens"] is not None else (r_tok + estimate_tokens(answer))
+            t_tok = extracted["total_tokens"] if extracted["total_tokens"] is not None else (p_tok + c_tok)
+            return {
+                "prompt_tokens": p_tok,
+                "completion_tokens": c_tok,
+                "reasoning_tokens": r_tok,
+                "total_tokens": t_tok,
+            }
+
+        p_tok = estimate_tokens(prompt)
+        r_tok = estimate_tokens(reasoning)
+        a_tok = estimate_tokens(answer)
+        c_tok = r_tok + a_tok
+        t_tok = p_tok + c_tok
+        return {
+            "prompt_tokens": p_tok,
+            "completion_tokens": c_tok,
+            "reasoning_tokens": r_tok,
+            "total_tokens": t_tok,
+        }
+
+    def _parse_model_output(self, raw_output: Any) -> tuple[str, str, Any]:
+        """Parse raw model output into reasoning CoT, final answer, and token metadata."""
+        if isinstance(raw_output, dict):
+            reasoning = ""
+            for k in ("reasoning", "cot", "thinking"):
+                if k in raw_output and raw_output[k] is not None:
+                    reasoning = str(raw_output[k])
+                    break
+            answer = ""
+            for k in ("answer", "response", "predicted_answer"):
+                if k in raw_output and raw_output[k] is not None:
+                    answer = str(raw_output[k])
+                    break
+            tu = raw_output.get("token_usage")
+            return reasoning, answer, tu
+
+        if hasattr(raw_output, "reasoning") or hasattr(raw_output, "answer") or hasattr(raw_output, "cot") or hasattr(raw_output, "response") or hasattr(raw_output, "predicted_answer"):
+            reasoning = ""
+            for attr in ("reasoning", "cot", "thinking"):
+                if hasattr(raw_output, attr) and getattr(raw_output, attr) is not None:
+                    reasoning = str(getattr(raw_output, attr))
+                    break
+            answer = ""
+            for attr in ("answer", "response", "predicted_answer"):
+                if hasattr(raw_output, attr) and getattr(raw_output, attr) is not None:
+                    answer = str(getattr(raw_output, attr))
+                    break
+            tu = getattr(raw_output, "token_usage", None)
+            return reasoning, answer, tu
+
+        text = str(raw_output or "").strip()
+
+        # Handle <think>...</think> tags
+        if "<think>" in text and "</think>" in text:
+            parts = text.split("</think>", 1)
+            reasoning = parts[0].replace("<think>", "").strip()
+            answer = parts[1].strip()
+            return reasoning, answer, {}
+
+        # Handle Reasoning: / Answer: markers
+        if "reasoning:" in text.lower() and "answer:" in text.lower():
+            r_idx = text.lower().find("reasoning:")
+            a_idx = text.lower().find("answer:")
+            if r_idx < a_idx:
+                reasoning = text[r_idx + 10 : a_idx].strip()
+                answer = text[a_idx + 7 :].strip()
+                return reasoning, answer, {}
+
+        # If line breaks exist, treat first part as reasoning and last line as answer
+        lines = [line.strip() for line in text.split("\n") if line.strip()]
+        if len(lines) > 1:
+            reasoning = "\n".join(lines[:-1])
+            answer = lines[-1]
+            return reasoning, answer, {}
+
+        return text, text, {}
+
+
+    def _invoke_model(self, model: Any, prompt: str, language: str) -> Any:
+        """Call model using appropriate signature (generate, predict, or call)."""
+        if callable(model):
+            if _accepts_language(model):
+                return model(prompt, language=language)
+            return model(prompt)
+
+        if hasattr(model, "generate") and callable(model.generate):
+            if _accepts_language(model.generate):
+                return model.generate(prompt, language=language)
+            return model.generate(prompt)
+
+        if hasattr(model, "predict") and callable(model.predict):
+            if _accepts_language(model.predict):
+                return model.predict(prompt, language=language)
+            return model.predict(prompt)
+
+        raise ValueError(f"Model object {type(model)} is not callable and lacks generate/predict methods.")
+
+    def evaluate_sample(self, model: Any, sample: dict[str, Any]) -> dict[str, Any]:
+        """Evaluate a single dataset sample."""
+        lang_raw = None
+        for k in ("language", "target_language", "lang"):
+            if k in sample and sample[k] is not None:
+                lang_raw = sample[k]
+                break
+        language = normalize_language(str(lang_raw) if lang_raw is not None else "English")
+
+        prompt = ""
+        for k in ("prompt", "question", "input"):
+            if k in sample and sample[k] is not None:
+                prompt = str(sample[k])
+                break
+
+        reference_answer = ""
+        for k in ("reference_answer", "expected_answer", "ground_truth", "target", "answer"):
+            if k in sample and sample[k] is not None:
+                reference_answer = str(sample[k])
+                break
+
+        raw_output = self._invoke_model(model, prompt, language)
+        reasoning, answer, tu_raw = self._parse_model_output(raw_output)
+
+        cot_fidelity = self.evaluate_cot_fidelity(reasoning, language)
+        accuracy = self.evaluate_accuracy(answer, reference_answer)
+        token_usage = self.compute_token_usage(prompt, reasoning, answer, tu_raw or raw_output)
+
+        return {
+            "language": language,
+            "prompt": prompt,
+            "reference_answer": reference_answer,
+            "reasoning": reasoning,
+            "predicted_answer": answer,
+            "cot_fidelity": cot_fidelity,
+            "accuracy": accuracy,
+            "token_usage": token_usage,
+        }
+
+    def compute_transfer_efficiency(self, by_language_metrics: dict[str, dict[str, Any]]) -> dict[str, float]:
+        """Calculate cross-lingual transfer efficiency relative to English.
+
+        If English accuracy is missing or zero, the highest per-language accuracy
+        becomes the reference. If no positive reference exists, efficiency is 0.0.
+        """
+        raw_eng = by_language_metrics.get("English") if isinstance(by_language_metrics.get("English"), dict) else {}
+        english_acc = (raw_eng.get("accuracy") or 0.0) if raw_eng else 0.0
+        positive_accs = [
+            m["accuracy"]
+            for m in by_language_metrics.values()
+            if isinstance(m, dict) and m.get("accuracy") is not None and m.get("accuracy", 0.0) > 0
+        ]
+        reference_acc = english_acc if english_acc > 0 else (max(positive_accs) if positive_accs else 0.0)
+
+        efficiencies: dict[str, float] = {}
+        for lang, metrics in by_language_metrics.items():
+            acc = (metrics.get("accuracy") or 0.0) if isinstance(metrics, dict) else 0.0
+            if reference_acc > 0:
+                efficiencies[lang] = round(acc / reference_acc, 4)
+            else:
+                efficiencies[lang] = 0.0
+
+        return efficiencies
+
+    def evaluate(self, model: Any, dataset: Sequence[dict[str, Any]]) -> dict[str, Any]:
+        """Evaluate model on dataset and compile comprehensive report."""
+        def _empty_report() -> dict[str, Any]:
+            return {
+                "overall_accuracy": 0.0,
+                "overall_cot_fidelity": 0.0,
+                "overall_transfer_efficiency": 0.0,
+                "total_token_usage": {
+                    "prompt_tokens": 0,
+                    "completion_tokens": 0,
+                    "reasoning_tokens": 0,
+                    "total_tokens": 0,
+                },
+                "by_language": {},
+                "num_samples": 0,
+            }
+
+        if not dataset:
+            return _empty_report()
+
+        if self.target_languages:
+            target_langs = set(self.target_languages)
+            dataset = [
+                s for s in dataset
+                if normalize_language(
+                    s.get("language") or s.get("target_language") or s.get("lang") or "English"
+                ) in target_langs
+            ]
+
+        if not dataset:
+            return _empty_report()
+
+        sample_results = []
+        for sample in dataset:
+            try:
+                sample_results.append(self.evaluate_sample(model, sample))
+            except Exception as e:
+                lang_raw = sample.get("language") or sample.get("target_language") or sample.get("lang") or "English"
+                sample_results.append({
+                    "language": normalize_language(str(lang_raw)),
+                    "prompt": str(sample.get("prompt") or sample.get("question") or sample.get("input") or ""),
+                    "reference_answer": str(sample.get("reference_answer") or sample.get("expected_answer") or ""),
+                    "reasoning": "",
+                    "predicted_answer": "",
+                    "cot_fidelity": 0.0,
+                    "accuracy": 0.0,
+                    "error": str(e),
+                    "token_usage": {"prompt_tokens": 0, "completion_tokens": 0, "reasoning_tokens": 0, "total_tokens": 0},
+                })
+
+        if not sample_results:
+            return _empty_report()
+        by_lang_samples: dict[str, list[dict[str, Any]]] = {}
+        for res in sample_results:
+            lang = res["language"]
+            by_lang_samples.setdefault(lang, []).append(res)
+
+        by_language_metrics: dict[str, dict[str, Any]] = {}
+        tot_p_tokens = 0
+        tot_c_tokens = 0
+        tot_r_tokens = 0
+        tot_t_tokens = 0
+
+        for lang, samples in by_lang_samples.items():
+            cnt = len(samples)
+            acc = sum(s["accuracy"] for s in samples) / cnt
+            fid = sum(s["cot_fidelity"] for s in samples) / cnt
+
+            p_tok = sum(s["token_usage"]["prompt_tokens"] for s in samples)
+            c_tok = sum(s["token_usage"]["completion_tokens"] for s in samples)
+            r_tok = sum(s["token_usage"]["reasoning_tokens"] for s in samples)
+            t_tok = sum(s["token_usage"]["total_tokens"] for s in samples)
+
+            tot_p_tokens += p_tok
+            tot_c_tokens += c_tok
+            tot_r_tokens += r_tok
+            tot_t_tokens += t_tok
+
+            by_language_metrics[lang] = {
+                "sample_count": cnt,
+                "accuracy": round(acc, 4),
+                "cot_fidelity": round(fid, 4),
+                "token_usage": {
+                    "prompt_tokens": p_tok,
+                    "completion_tokens": c_tok,
+                    "reasoning_tokens": r_tok,
+                    "total_tokens": t_tok,
+                },
+            }
+
+        transfer_efficiencies = self.compute_transfer_efficiency(by_language_metrics)
+        for lang, eff in transfer_efficiencies.items():
+            by_language_metrics[lang]["transfer_efficiency"] = eff
+
+        non_english_effs = [
+            eff for lang, eff in transfer_efficiencies.items() if lang != "English"
+        ]
+        overall_transfer_eff = (
+            round(sum(non_english_effs) / len(non_english_effs), 4)
+            if non_english_effs
+            else 1.0
+        )
+
+        overall_accuracy = round(sum(s["accuracy"] for s in sample_results) / len(sample_results), 4)
+        overall_fidelity = round(sum(s["cot_fidelity"] for s in sample_results) / len(sample_results), 4)
+
+        return {
+            "overall_accuracy": overall_accuracy,
+            "overall_cot_fidelity": overall_fidelity,
+            "overall_transfer_efficiency": overall_transfer_eff,
+            "total_token_usage": {
+                "prompt_tokens": tot_p_tokens,
+                "completion_tokens": tot_c_tokens,
+                "reasoning_tokens": tot_r_tokens,
+                "total_tokens": tot_t_tokens,
+            },
+            "by_language": by_language_metrics,
+            "num_samples": len(sample_results),
+        }
+
+
+def run_evaluation(
+    model: Any, dataset: Sequence[dict[str, Any]], target_languages: Optional[Sequence[str]] = None
+) -> dict[str, Any]:
+    """Entrypoint function to run multilingual reasoning evaluation."""
+    evaluator = MultilingualReasoningEvaluator(target_languages=target_languages)
+    return evaluator.evaluate(model, dataset)

chapter8/MultilingualReasoning/gpt_oss_20b_sft.py

@@ -0,0 +1,554 @@
+"""
+多语言推理模型微调脚本
+
+本脚本展示如何使用 Hugging Face 的 TRL 库对 OpenAI 的 gpt-oss-20b 模型进行微调，
+使其能够在多种语言中进行有效推理。
+
+基于 OpenAI Cookbook 教程：
+https://cookbook.openai.com/articles/gpt-oss/fine-tune-transfomers
+
+作者: Edward Beeching, Quentin Gallouédec, Lewis Tunstall
+修改: 适配为完整的 Python 脚本
+
+⚠️  硬件要求（重要！）:
+- GPU: H100（80GB 显存）或更高配置
+- 训练时间: H100 上约 18 分钟
+- 使用 Mxfp4Config 量化和 LoRA 进行内存高效训练
+
+功能特性:
+- 使用 Mxfp4Config（针对 OpenAI 模型优化的 4-bit 浮点格式）
+- 使用 LoRA 进行内存高效的微调（包括 MoE 专家层）
+- 支持多语言推理（英语、西班牙语、法语、德语、意大利语等）
+- 可以混合语言（用一种语言提问，用另一种语言推理）
+- 所有超参数与 OpenAI Cookbook 教程完全一致
+"""
+
+import os
+import argparse
+try:
+    import torch
+    from datasets import load_dataset
+    from transformers import AutoModelForCausalLM, AutoTokenizer, Mxfp4Config
+    from peft import LoraConfig, PeftModel, get_peft_model
+    from trl import SFTTrainer, SFTConfig
+except ImportError:
+    torch = None
+    load_dataset = None
+    AutoModelForCausalLM = AutoTokenizer = Mxfp4Config = None
+    LoraConfig = PeftModel = get_peft_model = None
+    SFTTrainer = SFTConfig = None
+
+
+# ============================================================================
+# 第一部分：数据集准备
+# ============================================================================
+
+def load_and_prepare_dataset():
+    """
+    加载并准备多语言推理数据集
+    
+    使用 HuggingFaceH4/Multilingual-Thinking 数据集，该数据集包含：
+    - 多种语言的推理链（思维链）
+    - 支持英语、西班牙语、法语、德语、意大利语等
+    
+    Returns:
+        Dataset: 格式化后的训练数据集
+    """
+    print("=" * 80)
+    print("步骤 1: 加载数据集")
+    print("=" * 80)
+    
+    # 从 Hugging Face Hub 加载数据集
+    dataset = load_dataset("HuggingFaceH4/Multilingual-Thinking")
+    
+    print(f"数据集加载完成！")
+    print(f"训练样本数: {len(dataset['train'])}")
+    print(f"数据集列: {dataset['train'].column_names}")
+    print(f"\n示例数据:")
+    print(dataset['train'][0])
+    
+    return dataset['train']
+
+
+def format_chat_template(example, tokenizer):
+    """
+    格式化对话模板
+    
+    将数据集中的消息格式化为模型可以理解的对话格式
+    
+    Args:
+        example: 数据集中的一个样本
+        tokenizer: 分词器
+        
+    Returns:
+        dict: 格式化后的样本
+    """
+    # 应用聊天模板（带 messages 类型校验）
+    messages = example.get("messages")
+    if not isinstance(messages, list):
+        messages = []
+    example["text"] = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+    )
+    return example
+
+
+# ============================================================================
+# 第二部分：模型准备
+# ============================================================================
+
+def load_base_model(model_name="openai/gpt-oss-20b"):
+    """
+    加载基础模型和分词器
+    
+    使用 Mxfp4Config 进行量化，这是专门为 OpenAI 模型优化的 4-bit 浮点格式。
+    
+    Args:
+        model_name: 模型名称或路径
+        
+    Returns:
+        tuple: (model, tokenizer)
+    """
+    print("\n" + "=" * 80)
+    print("步骤 2: 加载基础模型")
+    print("=" * 80)
+    
+    # 加载分词器
+    print(f"加载分词器: {model_name}")
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    
+    # 设置 pad token（如果不存在）
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+    
+    # 配置 Mxfp4 量化（针对 OpenAI 模型优化）
+    print("使用 Mxfp4Config 量化...")
+    quantization_config = Mxfp4Config(dequantize=True)
+    
+    # 配置模型加载参数
+    model_kwargs = {
+        "attn_implementation": "eager",      # 注意力实现方式
+        "torch_dtype": torch.bfloat16,       # 使用 bfloat16 提高效率
+        "quantization_config": quantization_config,  # Mxfp4 量化配置
+        "use_cache": False,                   # 训练时禁用 KV 缓存
+        "device_map": "auto",                 # 自动分配设备
+    }
+    
+    # 加载模型
+    print(f"加载模型: {model_name}")
+    print("这可能需要几分钟时间...")
+    model = AutoModelForCausalLM.from_pretrained(model_name, **model_kwargs)
+    
+    print(f"模型加载完成！")
+    print(f"模型参数量: {model.num_parameters() / 1e9:.2f}B")
+    
+    return model, tokenizer
+
+
+def prepare_model_for_lora(model, lora_rank=8, lora_alpha=16):
+    """
+    配置 LoRA（低秩适应）进行高效微调
+    
+    LoRA 只训练少量参数，大大减少内存使用和训练时间。
+    针对 openai/gpt-oss-20b 的 MoE（混合专家）架构，除了注意力层外，
+    还需要特别指定 MLP 专家层进行训练。
+    
+    Args:
+        model: 基础模型
+        lora_rank: LoRA 秩（默认 8，与官方教程一致）
+        lora_alpha: LoRA 缩放参数（默认 16）
+        
+    Returns:
+        PeftModel: 配置了 LoRA 的模型
+    """
+    print("\n" + "=" * 80)
+    print("步骤 3: 配置 LoRA")
+    print("=" * 80)
+    
+    # LoRA 配置（与 OpenAI Cookbook 一致）
+    peft_config = LoraConfig(
+        r=lora_rank,                     # LoRA 秩
+        lora_alpha=lora_alpha,           # LoRA 缩放参数
+        target_modules="all-linear",     # 目标所有线性层
+        target_parameters=[              # MoE 专家层的特定参数
+            "7.mlp.experts.gate_up_proj",
+            "7.mlp.experts.down_proj",
+            "15.mlp.experts.gate_up_proj",
+            "15.mlp.experts.down_proj",
+            "23.mlp.experts.gate_up_proj",
+            "23.mlp.experts.down_proj",
+        ],
+    )
+    
+    print("LoRA 配置:")
+    print(f"  - Rank: {lora_rank}")
+    print(f"  - Alpha: {lora_alpha}")
+    print(f"  - 目标模块: {peft_config.target_modules}")
+    print(f"  - MoE 专家层参数: {len(peft_config.target_parameters)} 个")
+    
+    # 应用 LoRA
+    model = get_peft_model(model, peft_config)
+    
+    # 打印可训练参数统计
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    total_params = sum(p.numel() for p in model.parameters())
+    trainable_percent = 100 * trainable_params / total_params
+    
+    print(f"\n可训练参数统计:")
+    print(f"  - 可训练参数: {trainable_params:,} ({trainable_percent:.2f}%)")
+    print(f"  - 总参数: {total_params:,}")
+    
+    return model
+
+
+# ============================================================================
+# 第三部分：训练
+# ============================================================================
+
+def train_model(model, tokenizer, dataset, output_dir="./gpt-oss-20b-multilingual-reasoner", 
+                batch_size=4, num_epochs=1, learning_rate=2e-4, max_seq_length=2048):
+    """
+    使用 SFTTrainer 训练模型
+    
+    Args:
+        model: 配置了 LoRA 的模型
+        tokenizer: 分词器
+        dataset: 训练数据集
+        output_dir: 输出目录
+        batch_size: 批次大小（根据 GPU 显存调整，默认 4）
+        num_epochs: 训练轮数（默认 1）
+        learning_rate: 学习率（默认 2e-4）
+        max_seq_length: 最大序列长度
+        
+    Returns:
+        SFTTrainer: 训练好的 trainer 对象
+    """
+    print("\n" + "=" * 80)
+    print("步骤 4: 开始训练")
+    print("=" * 80)
+    
+    # 训练参数配置（与 OpenAI Cookbook 完全一致）
+    training_args = SFTConfig(
+        learning_rate=learning_rate,
+        gradient_checkpointing=True,
+        num_train_epochs=num_epochs,
+        logging_steps=1,
+        per_device_train_batch_size=batch_size,
+        gradient_accumulation_steps=4,
+        max_length=max_seq_length,
+        warmup_ratio=0.03,
+        lr_scheduler_type="cosine_with_min_lr",
+        lr_scheduler_kwargs={"min_lr_rate": 0.1},
+        output_dir=output_dir,
+        report_to="trackio",  # 设为 "trackio" 以启用实验跟踪
+        push_to_hub=False,  # 设为 True 以自动推送到 Hub
+    )
+    
+    print("训练配置:")
+    print(f"  - 批次大小: {batch_size}")
+    print(f"  - 梯度累积步数: {training_args.gradient_accumulation_steps}")
+    print(f"  - 有效批次大小: {batch_size * training_args.gradient_accumulation_steps}")
+    print(f"  - 训练轮数: {num_epochs}")
+    print(f"  - 学习率: {learning_rate}")
+    print(f"  - 学习率调度: {training_args.lr_scheduler_type}")
+    print(f"  - 最大序列长度: {max_seq_length}")
+    print(f"  - 输出目录: {output_dir}")
+    
+    # 初始化 SFTTrainer
+    trainer = SFTTrainer(
+        model=model,
+        args=training_args,
+        train_dataset=dataset,
+        processing_class=tokenizer,
+    )
+    
+    # 开始训练
+    print("\n开始训练...")
+    print("⚠️  在 H100 GPU 上训练约需 18 分钟")
+    print("-" * 80)
+    
+    trainer.train()
+    
+    print("\n" + "=" * 80)
+    print("训练完成！")
+    print("=" * 80)
+    
+    return trainer
+
+
+# ============================================================================
+# 第四部分：保存和推送模型
+# ============================================================================
+
+def save_and_push_model(trainer, output_dir, push_to_hub=False, hub_model_id=None):
+    """
+    保存模型并可选择推送到 Hugging Face Hub
+    
+    Args:
+        trainer: 训练好的 trainer 对象
+        output_dir: 输出目录
+        push_to_hub: 是否推送到 Hub
+        hub_model_id: Hub 上的模型 ID
+    """
+    print("\n" + "=" * 80)
+    print("步骤 5: 保存模型")
+    print("=" * 80)
+    
+    # 保存模型到本地
+    print(f"保存模型到: {output_dir}")
+    trainer.save_model(output_dir)
+    print("模型保存完成！")
+    
+    # 可选：推送到 Hugging Face Hub
+    if push_to_hub:
+        if hub_model_id is None:
+            raise ValueError("需要提供 hub_model_id 才能推送到 Hub")
+
+        print(f"\n推送模型到 Hugging Face Hub: {hub_model_id}")
+        # Trainer.push_to_hub 从 args.hub_model_id 取仓库名；不设置的话会
+        # 忽略用户传入的 --hub_model_id，推到 output_dir 同名的默认仓库。
+        trainer.args.hub_model_id = hub_model_id
+        trainer.push_to_hub(
+            dataset_name="HuggingFaceH4/Multilingual-Thinking",
+        )
+        print("模型已成功推送到 Hub！")
+
+
+# ============================================================================
+# 第五部分：推理
+# ============================================================================
+
+def load_trained_model(base_model_name, peft_model_path):
+    """
+    加载训练好的模型进行推理
+    
+    Args:
+        base_model_name: 基础模型名称
+        peft_model_path: LoRA 权重路径
+        
+    Returns:
+        tuple: (model, tokenizer)
+    """
+    print("\n" + "=" * 80)
+    print("加载训练好的模型进行推理")
+    print("=" * 80)
+    
+    # 加载分词器
+    print(f"加载分词器: {base_model_name}")
+    tokenizer = AutoTokenizer.from_pretrained(base_model_name)
+    
+    # 加载基础模型
+    print(f"加载基础模型: {base_model_name}")
+    model_kwargs = {
+        "attn_implementation": "eager",
+        "torch_dtype": "auto",
+        "use_cache": True,  # 推理时启用 KV 缓存
+        "device_map": "auto",
+    }
+    base_model = AutoModelForCausalLM.from_pretrained(base_model_name, **model_kwargs)
+    
+    # 加载并合并 LoRA 权重
+    print(f"加载 LoRA 权重: {peft_model_path}")
+    model = PeftModel.from_pretrained(base_model, peft_model_path)
+    
+    print("合并 LoRA 权重与基础模型...")
+    model = model.merge_and_unload()
+    
+    print("模型加载完成！")
+    
+    return model, tokenizer
+
+
+def generate_response(model, tokenizer, reasoning_language, user_prompt, 
+                     max_new_tokens=512, temperature=0.6, format_output=True):
+    """
+    生成多语言推理响应
+    
+    Args:
+        model: 训练好的模型
+        tokenizer: 分词器
+        reasoning_language: 推理使用的语言
+        user_prompt: 用户提问
+        max_new_tokens: 最大生成 token 数
+        temperature: 采样温度（越高越随机）
+        format_output: 是否格式化输出（使用明显的标记）
+        
+    Returns:
+        str: 生成的完整响应
+    """
+    # 构建消息
+    system_prompt = f"reasoning language: {reasoning_language}"
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_prompt},
+    ]
+    
+    # 应用聊天模板
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt",
+    ).to(model.device)
+    
+    # 生成配置
+    gen_kwargs = {
+        "max_new_tokens": max_new_tokens,
+        "do_sample": True,
+        "temperature": temperature,
+        "top_p": None,
+        "top_k": None,
+    }
+    
+    # 生成响应
+    print(f"\n生成响应...")
+    print(f"推理语言: {reasoning_language}")
+    print(f"用户提问: {user_prompt}")
+    
+    with torch.no_grad():
+        output_ids = model.generate(input_ids, **gen_kwargs)
+    
+    # 解码输出 - 保留特殊标记以便解析
+    response_with_tokens = tokenizer.batch_decode(output_ids, skip_special_tokens=False)[0]
+    print("-" * 80)
+    print(response_with_tokens)
+    print("-" * 80)
+
+
+def run_inference_examples(model, tokenizer):
+    """
+    运行多个推理示例
+    
+    Args:
+        model: 训练好的模型
+        tokenizer: 分词器
+    """
+    print("\n" + "=" * 80)
+    print("推理示例")
+    print("=" * 80)
+    
+    # 示例 1: 西班牙语提问，德语推理
+    print("\n[示例 1: 西班牙语提问 + 德语推理]")
+    generate_response(
+        model, tokenizer,
+        reasoning_language="German",
+        user_prompt="¿Cuál es el capital de Australia?",  # 澳大利亚的首都是什么？
+        format_output=True,
+    )
+    
+    # 示例 2: 英语提问，中文推理
+    print("\n\n[示例 2: 英语提问 + 中文推理]")
+    generate_response(
+        model, tokenizer,
+        reasoning_language="Chinese",
+        user_prompt="What is the national symbol of Canada?",
+        format_output=True,
+    )
+    
+    # 示例 3: 中文提问，中文推理
+    print("\n\n[示例 3: 中文提问 + 中文推理]")
+    generate_response(
+        model, tokenizer,
+        reasoning_language="Chinese",
+        user_prompt="求解 x^2 - 2x + 1 = 0 的根",
+        format_output=True,
+    )
+
+
+# ============================================================================
+# 主函数
+# ============================================================================
+
+def main():
+    """主函数：完整的训练流程"""
+    
+    parser = argparse.ArgumentParser(description="多语言推理模型微调")
+    parser.add_argument(
+        "--mode", 
+        type=str, 
+        choices=["train", "inference", "full"],
+        default="full",
+        help="运行模式: train（仅训练）, inference（仅推理）, full（完整流程）"
+    )
+    parser.add_argument("--model_name", type=str, default="openai/gpt-oss-20b", help="基础模型名称")
+    parser.add_argument("--output_dir", type=str, default="./gpt-oss-20b-multilingual-reasoner", help="输出目录")
+    parser.add_argument("--batch_size", type=int, default=4, help="训练批次大小（默认 4，与官方教程一致）")
+    parser.add_argument("--num_epochs", type=int, default=1, help="训练轮数（默认 1，与官方教程一致）")
+    parser.add_argument("--learning_rate", type=float, default=2e-4, help="学习率（默认 2e-4，与官方教程一致）")
+    parser.add_argument("--max_seq_length", type=int, default=2048, help="最大序列长度")
+    parser.add_argument("--lora_rank", type=int, default=8, help="LoRA 秩（默认 8，与官方教程一致）")
+    parser.add_argument("--lora_alpha", type=int, default=16, help="LoRA alpha")
+    parser.add_argument("--push_to_hub", action="store_true", default=False, help="推送模型到 Hugging Face Hub")
+    parser.add_argument("--hub_model_id", type=str, default=None, help="Hub 模型 ID")
+    
+    args = parser.parse_args()
+    
+    print("=" * 80)
+    print("多语言推理模型微调")
+    print("=" * 80)
+    print(f"模式: {args.mode}")
+    print(f"基础模型: {args.model_name}")
+    print(f"输出目录: {args.output_dir}")
+    
+    # 训练模式
+    if args.mode in ["train", "full"]:
+        # 1. 加载数据集
+        dataset = load_and_prepare_dataset()
+        
+        # 2. 加载基础模型（使用 Mxfp4Config 量化）
+        model, tokenizer = load_base_model(args.model_name)
+        
+        # 3. 配置 LoRA
+        model = prepare_model_for_lora(model, args.lora_rank, args.lora_alpha)
+        
+        # 4. 训练模型
+        trainer = train_model(
+            model, 
+            tokenizer, 
+            dataset,
+            output_dir=args.output_dir,
+            batch_size=args.batch_size,
+            num_epochs=args.num_epochs,
+            learning_rate=args.learning_rate,
+            max_seq_length=args.max_seq_length,
+        )
+        
+        # 5. 保存模型
+        save_and_push_model(
+            trainer, 
+            args.output_dir,
+            push_to_hub=args.push_to_hub,
+            hub_model_id=args.hub_model_id,
+        )
+        
+        if args.mode == "full":
+            # full 模式继续跑推理：先释放训练占用的显存，
+            # 再按推理路径从 output_dir 重新加载已保存的模型。
+            del trainer
+            del model
+            torch.cuda.empty_cache()
+            print("\n训练完成！已释放训练显存，继续运行推理示例。")
+        else:
+            print("\n训练完成！建议重启内核以释放 GPU 显存后再进行推理。")
+
+    # 推理模式（inference 单独运行；full 在训练后接着运行）
+    if args.mode in ["inference", "full"]:
+        if not os.path.exists(args.output_dir):
+            print(f"错误: 未找到模型目录 {args.output_dir}")
+            print("请先运行训练或指定正确的模型路径")
+            return
+        
+        # 加载训练好的模型
+        model, tokenizer = load_trained_model(args.model_name, args.output_dir)
+        
+        # 运行推理示例
+        run_inference_examples(model, tokenizer)
+    
+    print("\n" + "=" * 80)
+    print("完成！")
+    print("=" * 80)
+
+
+if __name__ == "__main__":
+    main()
+

chapter8/MultilingualReasoning/requirements.txt

@@ -0,0 +1,21 @@
+# 多语言推理模型微调依赖
+
+# 核心依赖
+torch>=2.0.0
+transformers>=4.55.0
+datasets>=2.14.0
+accelerate>=0.20.0
+
+# 微调相关
+trl>=0.20.0
+peft>=0.17.0
+bitsandbytes>=0.41.0
+
+# 工具和监控
+trackio
+huggingface-hub>=0.16.0
+
+# 可选：实验跟踪
+# wandb
+# tensorboard
+