ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Repository utility scripts exposed as a Python package for tests and tooling."""
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env bash
|
||||
# Assemble the MkDocs docs directory (`_web/`) from the book Markdown sources.
|
||||
# Reader-facing Markdown, images, frontend assets, and linked JSON evidence are
|
||||
# copied; code, PDFs and LaTeX sources are left out so the generated site stays
|
||||
# small. The original sources are never modified.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DEST="$ROOT/_web"
|
||||
|
||||
rm -rf "$DEST"
|
||||
mkdir -p "$DEST"
|
||||
|
||||
# Site homepage (root index.md).
|
||||
cp "$ROOT/index.md" "$DEST/index.md"
|
||||
|
||||
# Translated site homepages (root index.<lang>.md), when present. The
|
||||
# language switcher maps home <-> home for these editions (see
|
||||
# scripts/site_i18n.py, which lists them in the generated catalog).
|
||||
for home in "$ROOT"/index.*.md; do
|
||||
[ -f "$home" ] || continue
|
||||
cp "$home" "$DEST/$(basename "$home")"
|
||||
done
|
||||
|
||||
# robots.txt at the site root (points crawlers at the auto-generated sitemap).
|
||||
[ -f "$ROOT/robots.txt" ] && cp "$ROOT/robots.txt" "$DEST/robots.txt"
|
||||
|
||||
# The language editions, each with its images/ subfolder.
|
||||
for lang in book book-en book-es book-id book-ru book-ta book-vi book-zhtw book-ja book-ar book-tr book-ko book-hu book-he; do
|
||||
mkdir -p "$DEST/$lang"
|
||||
cp -R "$ROOT/$lang" "$DEST/"
|
||||
done
|
||||
|
||||
# Promote each chapter of the default (zh) edition to a directory index
|
||||
# (book/chapterN.md -> book/chapterN/index.md) so mkdocs.yml can use
|
||||
# navigation.indexes to attach the chapter prose to its nav section —
|
||||
# clicking a chapter title in the sidebar then opens the chapter directly.
|
||||
# The rendered URL is unchanged (/book/chapterN/, thanks to directory
|
||||
# URLs). The file now lives one directory deeper, so its relative image
|
||||
# references need a ../ prefix. Translated editions stay flat files: they
|
||||
# are not listed in the nav, so they gain nothing from the promotion.
|
||||
for n in 1 2 3 4 5 6 7 8 9 10; do
|
||||
src="$DEST/book/chapter$n.md"
|
||||
[ -f "$src" ] || continue
|
||||
mkdir -p "$DEST/book/chapter$n"
|
||||
sed \
|
||||
-e 's|](images/|](../images/|g' \
|
||||
-e 's|](../chapter|](../../chapter|g' \
|
||||
"$src" > "$DEST/book/chapter$n/index.md"
|
||||
rm "$src"
|
||||
done
|
||||
|
||||
# The companion experiment directories (chapterN/). Each chapter has a
|
||||
# README.md (experiment index) plus one subfolder per experiment, also
|
||||
# documented by its own README.md. These are exposed under /chapterN/ so
|
||||
# readers can step from the chapter prose straight into runnable code.
|
||||
for ch in chapter1 chapter2 chapter3 chapter4 chapter5 \
|
||||
chapter6 chapter7 chapter8 chapter9 chapter10; do
|
||||
if [ -d "$ROOT/$ch" ]; then
|
||||
cp -R "$ROOT/$ch" "$DEST/"
|
||||
fi
|
||||
done
|
||||
|
||||
# Copy site-level assets (JS/CSS for the language switcher) that MkDocs
|
||||
# resolves relative to docs_dir.
|
||||
cp -R "$ROOT/extras" "$DEST/extras"
|
||||
|
||||
# Site-wide static assets — logo, favicon, social OG images. Referenced by
|
||||
# mkdocs.yml as `assets/<file>` (relative to docs_dir).
|
||||
if [ -d "$ROOT/assets" ]; then
|
||||
mkdir -p "$DEST/assets"
|
||||
cp -R "$ROOT/assets/." "$DEST/assets/"
|
||||
fi
|
||||
|
||||
# Keep reader-facing site assets, including JSON experiment evidence linked
|
||||
# from chapter documentation. The helper is tested independently so changes to
|
||||
# the publication allowlist do not silently introduce broken links.
|
||||
python3 "$ROOT/scripts/clean_site_files.py" "$DEST"
|
||||
|
||||
# Drop bulk data files that some experiments bundle as their dataset but
|
||||
# that don't belong in the reading site (hundreds of legal-doc markdown
|
||||
# files would also slow the git-revision-date plugin to a crawl).
|
||||
rm -rf \
|
||||
"$DEST/chapter3/contextual-retrieval/laws" \
|
||||
"$DEST/chapter3/agentic-rag/laws" \
|
||||
2>/dev/null || true
|
||||
|
||||
# Vendored JavaScript dependencies can contain thousands of their own
|
||||
# Markdown files. They are irrelevant to the book site and make MkDocs scan
|
||||
# needlessly large directory trees after the file-type cleanup above.
|
||||
find "$DEST" -type d -name node_modules -prune -exec rm -rf {} +
|
||||
|
||||
# Rewrite the relative links used inside the experiment READMEs so they
|
||||
# resolve correctly in the MkDocs site. Source files are NOT modified —
|
||||
# only the copies under _web/.
|
||||
#
|
||||
# The README source uses GitHub-style relative paths that don't survive
|
||||
# MkDocs rendering. Two patterns appear in chapter index pages
|
||||
# (`chapterN/README.md`):
|
||||
# ../book/chapter1.md (point at the chapter prose)
|
||||
# ../README.md (point at the repo root / homepage)
|
||||
#
|
||||
# MkDocs renders pages as directory URLs (`chapter1/`), so the `.md`
|
||||
# suffix must be stripped. Keep the paths RELATIVE (no leading slash) so
|
||||
# they keep working under the site's sub-path
|
||||
# (`https://bojieli.github.io/ai-agent-book/`).
|
||||
find "$DEST/chapter"* -type f -name '*.md' -print0 \
|
||||
| xargs -0 sed -i.bak \
|
||||
-e 's|\.\./book/\([a-zA-Z0-9_-]*\)\.md|../book/\1/|g' \
|
||||
-e 's|\.\./README\.md|../|g'
|
||||
# macOS sed needs the backup suffix above; clean up the .bak files.
|
||||
find "$DEST" -name '*.md.bak' -delete
|
||||
|
||||
# Per-language experiment index pages (chapterN/README.<lang>.md) contain
|
||||
# relative links like [exp](local_llm_serving/) that resolve correctly on
|
||||
# the Chinese URL /chapterN/ but break on the translated URL
|
||||
# /chapterN/README.<lang>/ (they'd resolve to /chapterN/README.<lang>/exp/,
|
||||
# which 404s). Rewrite those relative links to be relative to /chapterN/
|
||||
# by prefixing ../ — this makes them resolve to /chapterN/<exp>/ in any
|
||||
# language edition.
|
||||
#
|
||||
# Only touches README.<lang>.md (not README.md, where the links already work),
|
||||
# and only relative links that don't start with . / # http or contain :
|
||||
#
|
||||
# The "back to main README" links ](../docs/<locale>/README.md) point into
|
||||
# docs/, which is never copied into the site's docs_dir — MkDocs leaves the
|
||||
# raw href and it 404s. Map them to ../../ (the site home) instead.
|
||||
find "$DEST/chapter"* -type f -name 'README.[a-zA-Z-]*.md' -print0 \
|
||||
| xargs -0 sed -i.bak -E \
|
||||
-e 's|\]\(([a-zA-Z][a-zA-Z0-9_-]*)/\)|](../\1/)|g' \
|
||||
-e 's|\]\(\.\./docs/[a-zA-Z-]+/README\.md\)|](../../)|g'
|
||||
find "$DEST" -name '*.md.bak' -delete
|
||||
|
||||
echo "Assembled docs into $DEST"
|
||||
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
"""检查多语言版本的结构完整性。
|
||||
|
||||
防止主页或某章 README 改动后，其它语言版本跟不上而漂移。CI 中运行；
|
||||
本地也可直接 `python scripts/check_i18n_consistency.py` 跑。
|
||||
|
||||
核心原则：**自动发现语言，不硬编码**。下次有人加新语言（日语、韩语…）时，
|
||||
CI 自动适配，无需改脚本。
|
||||
|
||||
目录约定（中文为主语言）：
|
||||
- 中文主 README：仓库根目录 README.md（不放进 docs/）
|
||||
- 其它语言主 README：docs/<locale>/README.md（如 docs/en/README.md）
|
||||
- 学习建议：docs/<locale>/LEARNING.md（含中文 docs/zh-CN/LEARNING.md）
|
||||
- 章节 README：中文默认 chapterN/README.md；其它语言 chapterN/README.<locale>.md
|
||||
（ISO 639-1 + ISO 3166-1，如 README.en.md、README.zh-TW.md）
|
||||
|
||||
严格规则：**只要某语言有自己的主 README，CI 就要求它完整**：
|
||||
- 10 章 chapter README（中文为 README.md，其它为 README.<locale>.md）
|
||||
- docs/<locale>/LEARNING.md
|
||||
- 每章项目数与中文版对齐
|
||||
- git clone 命令数对齐
|
||||
- 内容速览表 ≥5 列
|
||||
|
||||
退出码：0 = 全部一致；1 = 发现不一致。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
CHAPTERS = range(1, 11)
|
||||
|
||||
|
||||
def chapter_suffix(locale: str) -> str:
|
||||
"""章节 README 后缀：中文（zh-CN）默认为空 → README.md；其它如 en → .en。"""
|
||||
if locale == "zh-CN":
|
||||
return ""
|
||||
return f".{locale}"
|
||||
|
||||
|
||||
def main_readme_path(locale: str) -> Path:
|
||||
if locale == "zh-CN":
|
||||
return ROOT / "README.md"
|
||||
return ROOT / "docs" / locale / "README.md"
|
||||
|
||||
|
||||
def learning_path(locale: str) -> Path:
|
||||
return ROOT / "docs" / locale / "LEARNING.md"
|
||||
|
||||
|
||||
def project_count_in_table(path: Path) -> int:
|
||||
"""统计 chapter README 表格里项目数据行数（含 ✅/📖/🚧 类型列的行）。"""
|
||||
if not path.exists():
|
||||
return -1
|
||||
pattern = re.compile(r"^\|.*\| [✅📖🚧]+ \|")
|
||||
return sum(
|
||||
1 for line in path.read_text(encoding="utf-8").splitlines() if pattern.match(line)
|
||||
)
|
||||
|
||||
|
||||
def count_git_clones(path: Path) -> int:
|
||||
if not path.exists():
|
||||
return -1
|
||||
return len(re.findall(r"^git clone ", path.read_text(encoding="utf-8"), re.MULTILINE))
|
||||
|
||||
|
||||
def toc_table_columns(path: Path) -> int:
|
||||
"""主 README 内容速览表第一个数据行的列数。"""
|
||||
if not path.exists():
|
||||
return -1
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if re.match(r"^\| \d+ \|", line):
|
||||
return line.count("|") - 1
|
||||
return -1
|
||||
|
||||
|
||||
def discover_locales() -> list[str]:
|
||||
"""发现所有主语言 locale。
|
||||
|
||||
- 中文（zh-CN）始终包含（根目录 README.md）
|
||||
- 其余：docs/<locale>/README.md 存在即纳入
|
||||
"""
|
||||
locales = ["zh-CN"]
|
||||
docs = ROOT / "docs"
|
||||
if docs.is_dir():
|
||||
for path in sorted(docs.iterdir()):
|
||||
if path.is_dir() and path.name != "zh-CN" and (path / "README.md").exists():
|
||||
locales.append(path.name)
|
||||
return locales
|
||||
|
||||
|
||||
def main() -> int:
|
||||
errors: list[str] = []
|
||||
|
||||
# ===== 自动发现语言 =====
|
||||
locales = discover_locales()
|
||||
|
||||
print("== 自动发现语言 ==")
|
||||
print(f" 发现 {len(locales)} 个主 README（全部要求完整翻译）:")
|
||||
for locale in locales:
|
||||
print(f" {locale} (chapter suffix: {chapter_suffix(locale)!r})")
|
||||
print()
|
||||
|
||||
# ===== 检查 1：每个发现的主 README 都有完整结构 =====
|
||||
print("== 检查 1：主 README 内容速览表结构（≥5 列）==")
|
||||
for locale in locales:
|
||||
path = main_readme_path(locale)
|
||||
cols = toc_table_columns(path)
|
||||
if cols < 5:
|
||||
errors.append(
|
||||
f"{path.relative_to(ROOT)} ({locale}) 内容速览表列数 {cols} < 5"
|
||||
"（应至少 5 列：章/主题/核心/正文/代码）"
|
||||
)
|
||||
else:
|
||||
print(f" ✓ {locale}: {cols} 列")
|
||||
print()
|
||||
|
||||
# ===== 检查 2：git clone 命令数对齐（以中文版为基准）=====
|
||||
print("== 检查 2：主 README git clone 命令数 ==")
|
||||
zh_clones = count_git_clones(main_readme_path("zh-CN"))
|
||||
print(f" 中文基准：{zh_clones} 条")
|
||||
for locale in locales:
|
||||
if locale == "zh-CN":
|
||||
continue
|
||||
path = main_readme_path(locale)
|
||||
count = count_git_clones(path)
|
||||
if count != zh_clones:
|
||||
errors.append(
|
||||
f"{path.relative_to(ROOT)} ({locale}) git clone 数 {count} ≠ 中文版 {zh_clones}"
|
||||
)
|
||||
else:
|
||||
print(f" ✓ {locale}: {count} 条")
|
||||
print()
|
||||
|
||||
# ===== 检查 3：每个主语言必须有 docs/<locale>/LEARNING.md =====
|
||||
print("== 检查 3：docs/<locale>/LEARNING.md 齐全 ==")
|
||||
for locale in locales:
|
||||
path = learning_path(locale)
|
||||
if not path.exists():
|
||||
errors.append(
|
||||
f"{path.relative_to(ROOT)} 不存在（{locale} 是主语言，需有学习建议文档）"
|
||||
)
|
||||
else:
|
||||
print(f" ✓ {path.relative_to(ROOT)} ({locale})")
|
||||
print()
|
||||
|
||||
# ===== 检查 4：每个主 README 语言必须有全部 10 章 README =====
|
||||
print("== 检查 4：chapterN/README[.locale].md 齐全 ==")
|
||||
for locale in locales:
|
||||
suffix = chapter_suffix(locale)
|
||||
missing = []
|
||||
for n in CHAPTERS:
|
||||
path = ROOT / f"chapter{n}/README{suffix}.md"
|
||||
if not path.exists():
|
||||
missing.append(str(n))
|
||||
if missing:
|
||||
errors.append(
|
||||
f"{locale} 缺章节 README：第 {', '.join(missing)} 章"
|
||||
)
|
||||
else:
|
||||
print(f" ✓ {locale}: 10 章齐全")
|
||||
print()
|
||||
|
||||
# ===== 检查 5：每章项目数对齐（所有主 README 语言）=====
|
||||
print("== 检查 5：每章项目数（所有语言对齐）==")
|
||||
zh_counts = {
|
||||
n: project_count_in_table(ROOT / f"chapter{n}/README.md")
|
||||
for n in CHAPTERS
|
||||
}
|
||||
total_zh = sum(zh_counts.values())
|
||||
print(f" 中文基准：{total_zh} 项目，分布 {[zh_counts[n] for n in CHAPTERS]}")
|
||||
for locale in locales:
|
||||
if locale == "zh-CN":
|
||||
continue
|
||||
suffix = chapter_suffix(locale)
|
||||
total = 0
|
||||
mismatches = []
|
||||
for n in CHAPTERS:
|
||||
path = ROOT / f"chapter{n}/README{suffix}.md"
|
||||
count = project_count_in_table(path)
|
||||
total += max(count, 0)
|
||||
zh = zh_counts[n]
|
||||
if count != zh:
|
||||
mismatches.append(f"第{n}章 {count}≠{zh}")
|
||||
if mismatches:
|
||||
errors.append(
|
||||
f"{locale} 项目数不一致（{len(mismatches)} 处）：{'; '.join(mismatches[:3])}"
|
||||
)
|
||||
else:
|
||||
print(f" ✓ {locale}: {total} 项目对齐")
|
||||
print()
|
||||
|
||||
# ===== 检查 6：主 README 语言切换栏完整性 =====
|
||||
print("== 检查 6：主 README 语言切换栏列出所有语言 ==")
|
||||
zh_text = main_readme_path("zh-CN").read_text(encoding="utf-8")
|
||||
switcher_match = re.search(
|
||||
r"\*\*[^*]*中文[^*]*\*\*.*?(?=\n\n|\n[^*])", zh_text, re.DOTALL
|
||||
)
|
||||
if switcher_match:
|
||||
switcher = switcher_match.group(0)
|
||||
missing_in_switcher = []
|
||||
for locale in locales:
|
||||
if locale == "zh-CN":
|
||||
continue
|
||||
# 语言切换栏应链接到 docs/<locale>/README.md
|
||||
if f"docs/{locale}/README.md" not in switcher:
|
||||
missing_in_switcher.append(locale)
|
||||
if missing_in_switcher:
|
||||
errors.append(
|
||||
f"README.md 语言切换栏缺少：{', '.join(missing_in_switcher)}"
|
||||
)
|
||||
else:
|
||||
print(f" ✓ README.md 列出全部 {len(locales)} 种语言")
|
||||
else:
|
||||
print(" ⚠️ 未找到语言切换栏（跳过此项检查）")
|
||||
print()
|
||||
|
||||
# ===== 汇总 =====
|
||||
if errors:
|
||||
print(f"❌ 发现 {len(errors)} 个问题：")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
print()
|
||||
print("修复提示：")
|
||||
print(" - 文件缺失：从中文版复制并翻译")
|
||||
print(" - 非中文主 README 放在 docs/<locale>/README.md")
|
||||
print(" - 学习建议放在 docs/<locale>/LEARNING.md")
|
||||
print(" - 项目数不一致：参考中文版 chapterN/README.md 同步项目列表")
|
||||
print(" - git clone 不一致：参考 README.md 附录段同步")
|
||||
print(" - 内容速览表结构：参考 README.md 的 5 列模板")
|
||||
print(" - 语言切换栏：参考 README.md 顶部，加入 docs/<locale>/README.md 链接")
|
||||
print(" - 章节 README 命名：中文为 README.md，其它为 README.<locale>.md（如 README.en.md）")
|
||||
return 1
|
||||
print("✓ 所有语言版本结构一致/完整")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Keep reader-facing site assets and JSON files linked from rendered Markdown."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
import markdown
|
||||
|
||||
ALWAYS_PUBLISHED_SUFFIXES = {
|
||||
".css",
|
||||
".jpeg",
|
||||
".jpg",
|
||||
".js",
|
||||
".md",
|
||||
".png",
|
||||
".svg",
|
||||
".txt",
|
||||
}
|
||||
|
||||
|
||||
class LinkCollector(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.targets: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
attribute = (
|
||||
"src"
|
||||
if tag in ("img", "script", "iframe", "source", "embed", "audio", "video")
|
||||
else "href"
|
||||
if tag in ("a", "link")
|
||||
else None
|
||||
)
|
||||
if attribute is None:
|
||||
return
|
||||
values = dict(attrs)
|
||||
if values.get(attribute):
|
||||
self.targets.append(values[attribute] or "")
|
||||
|
||||
|
||||
def rendered_links(text: str) -> list[str]:
|
||||
collector = LinkCollector()
|
||||
collector.feed(markdown.markdown(text, extensions=["fenced_code"]))
|
||||
return collector.targets
|
||||
|
||||
|
||||
def safe_regular_file(path: Path, root: Path) -> Path | None:
|
||||
try:
|
||||
resolved = path.resolve(strict=True)
|
||||
resolved.relative_to(root)
|
||||
except (FileNotFoundError, RuntimeError, ValueError):
|
||||
return None
|
||||
return resolved if resolved.is_file() else None
|
||||
|
||||
|
||||
def linked_json_files(root: Path) -> set[Path]:
|
||||
linked: set[Path] = set()
|
||||
for source in root.rglob("*.md"):
|
||||
markdown_file = safe_regular_file(source, root)
|
||||
if markdown_file is None:
|
||||
continue
|
||||
text = markdown_file.read_text(encoding="utf-8", errors="replace")
|
||||
for target in rendered_links(text):
|
||||
parsed = urlsplit(target)
|
||||
if parsed.scheme or parsed.netloc:
|
||||
continue
|
||||
decoded = unquote(parsed.path)
|
||||
if not decoded.lower().endswith(".json"):
|
||||
continue
|
||||
candidate = root / decoded.lstrip("/") if decoded.startswith("/") else source.parent / decoded
|
||||
resolved = safe_regular_file(candidate, root)
|
||||
if resolved is not None:
|
||||
linked.add(resolved)
|
||||
return linked
|
||||
|
||||
|
||||
def clean(root: Path) -> None:
|
||||
root = root.resolve(strict=True)
|
||||
keep_json = linked_json_files(root)
|
||||
for path in root.rglob("*"):
|
||||
if not (path.is_file() or path.is_symlink()):
|
||||
continue
|
||||
resolved = safe_regular_file(path, root)
|
||||
keep_regular_asset = resolved is not None and path.suffix.lower() in ALWAYS_PUBLISHED_SUFFIXES
|
||||
if keep_regular_asset or resolved in keep_json:
|
||||
continue
|
||||
path.unlink()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) != 2:
|
||||
raise SystemExit("usage: clean_site_files.py DEST")
|
||||
clean(Path(sys.argv[1]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate assets/og-card.png — the Open Graph / Twitter share card.
|
||||
|
||||
The card is a static 1200x630 PNG checked into the repo (regenerating it
|
||||
is only needed when the branding text changes). scripts/seo_meta.py
|
||||
references it as <site_url>/assets/og-card.png on every page.
|
||||
|
||||
Fonts are macOS system fonts (Hiragino Sans GB); run this on a Mac.
|
||||
PingFang would be preferred but is an on-demand download on modern macOS
|
||||
and often absent from /System/Library/Fonts.
|
||||
"""
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
W, H = 1200, 630
|
||||
|
||||
HIRA = "/System/Library/Fonts/Hiragino Sans GB.ttc" # index 0 = W3, 2 = W6
|
||||
|
||||
# fenix palette (keep in sync with extras/book-theme.css)
|
||||
INK, SOFT, MUTE = "#2c3e50", "#57606a", "#8b949e"
|
||||
GREEN, BORDER, BG_SOFT = "#42b983", "#eaecef", "#f6f8fa"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
img = Image.new("RGB", (W, H), "#ffffff")
|
||||
d = ImageDraw.Draw(img)
|
||||
|
||||
title_f = ImageFont.truetype(HIRA, 86, index=2)
|
||||
sub_f = ImageFont.truetype(HIRA, 33, index=0)
|
||||
mono_f = ImageFont.truetype(HIRA, 38, index=0)
|
||||
foot_f = ImageFont.truetype(HIRA, 26, index=0)
|
||||
|
||||
x = 96
|
||||
d.text((x, 128), "深入理解 AI Agent", font=title_f, fill=INK)
|
||||
d.text((x, 270), "设计原理与工程实践 · 一本完整开源的 AI Agent 技术书",
|
||||
font=sub_f, fill=SOFT)
|
||||
|
||||
# The book's core formula, in a flat code-chip
|
||||
fy, fh = 374, 84
|
||||
ftext = "Agent = LLM + 上下文 + 工具"
|
||||
fw = d.textlength(ftext, font=mono_f) + 72
|
||||
d.rounded_rectangle([x, fy, x + fw, fy + fh], radius=8,
|
||||
fill=BG_SOFT, outline=BORDER, width=2)
|
||||
d.text((x + 36, fy + 21), ftext, font=mono_f, fill=GREEN)
|
||||
|
||||
d.text((x, 528), "bojieli/ai-agent-book · 10 章正文 · 92 个配套实验 · 5 种语言",
|
||||
font=foot_f, fill=MUTE)
|
||||
d.rectangle([0, H - 10, W, H], fill=GREEN)
|
||||
|
||||
img.save("assets/og-card.png", optimize=True)
|
||||
print("wrote assets/og-card.png")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render this repo's star history as PNG images (light + dark variants).
|
||||
|
||||
Fetches stargazer timestamps from the GitHub REST API, drops everything
|
||||
before START_DATE, and draws a cumulative "stars over time" chart with a
|
||||
gradient fill. Output: assets/star-history-{light,dark}.png
|
||||
|
||||
Usage:
|
||||
python scripts/gen_star_history.py [--repo owner/name] [--refresh]
|
||||
[--start-date YYYY-MM-DD] [--out-dir DIR]
|
||||
|
||||
Auth: set GITHUB_TOKEN (or GH_TOKEN, or have an authenticated `gh` CLI).
|
||||
Unauthenticated requests work too but are rate-limited to 60/hour
|
||||
(~1 request per 100 stars). Timestamps are cached next to this script so
|
||||
style tweaks don't re-hit the API; pass --refresh to re-fetch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
|
||||
import matplotlib.dates as mdates
|
||||
import numpy as np
|
||||
from matplotlib import pyplot as plt
|
||||
from matplotlib.colors import LinearSegmentedColormap, to_rgba
|
||||
from matplotlib.ticker import FuncFormatter
|
||||
|
||||
REPO = "bojieli/ai-agent-book"
|
||||
START_DATE = "2026-07-15" # UTC; stars before this date are excluded
|
||||
CACHE = Path(__file__).with_name(".star-history-cache.json")
|
||||
|
||||
ACCENT = "#f5a623" # warm amber, reads well on both light and dark
|
||||
|
||||
THEMES = {
|
||||
"light": dict(bg="#ffffff", text="#1f2328", subtext="#6a737d", grid="#dfe3e8"),
|
||||
"dark": dict(bg="#0d1117", text="#e6edf3", subtext="#8b949e", grid="#272d35"),
|
||||
}
|
||||
|
||||
# Upper bound on x-axis labels. The real guarantee comes from measuring the
|
||||
# rendered labels (see thin_xticklabels); this just keeps the tick step sane.
|
||||
MAX_XTICKS = 12
|
||||
DAY_STEPS = (1, 2, 3, 7, 14) # days between ticks
|
||||
MONTH_STEPS = (1, 2, 3, 6)
|
||||
YEAR_STEPS = (1, 2, 5, 10)
|
||||
|
||||
|
||||
def get_token() -> str | None:
|
||||
for var in ("GITHUB_TOKEN", "GH_TOKEN"):
|
||||
if token := os.environ.get(var, "").strip():
|
||||
return token
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["gh", "auth", "token"], capture_output=True, text=True, timeout=10
|
||||
)
|
||||
if out.returncode == 0 and out.stdout.strip():
|
||||
return out.stdout.strip()
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_json(url: str, headers: dict, retries: int = 4) -> list:
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.load(resp)
|
||||
except Exception as exc:
|
||||
if attempt == retries - 1:
|
||||
raise
|
||||
wait = 2**attempt
|
||||
print(f"request failed ({exc}); retrying in {wait}s...", file=sys.stderr)
|
||||
time.sleep(wait)
|
||||
return [] # unreachable
|
||||
|
||||
|
||||
def fetch_starred_at(repo: str, refresh: bool) -> list[str]:
|
||||
"""Return sorted ISO-8601 UTC timestamps of every star event."""
|
||||
if CACHE.exists() and not refresh:
|
||||
print(f"using cached stargazers from {CACHE}", file=sys.stderr)
|
||||
return json.loads(CACHE.read_text())
|
||||
|
||||
headers = {
|
||||
"Accept": "application/vnd.github.star+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
"User-Agent": "gen-star-history",
|
||||
}
|
||||
if token := get_token():
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
starred: list[str] = []
|
||||
page = 1
|
||||
while True:
|
||||
url = f"https://api.github.com/repos/{repo}/stargazers?per_page=100&page={page}"
|
||||
data = get_json(url, headers)
|
||||
if not data:
|
||||
break
|
||||
starred.extend(item["starred_at"] for item in data)
|
||||
print(f"\rfetched {len(starred)} stargazers...", end="", file=sys.stderr)
|
||||
page += 1
|
||||
print(file=sys.stderr)
|
||||
|
||||
starred.sort()
|
||||
CACHE.write_text(json.dumps(starred))
|
||||
return starred
|
||||
|
||||
|
||||
def parse_iso_timestamp(s: str) -> datetime:
|
||||
"""Parse ISO-8601 timestamps (including fractional seconds and offsets) into UTC."""
|
||||
s = s.strip()
|
||||
if s.endswith("Z") or s.endswith("z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
dt = datetime.fromisoformat(s)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
else:
|
||||
dt = dt.astimezone(timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
def build_series(starred: list[str], start: datetime) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Cumulative star count per star event, cropped to `start` (UTC)."""
|
||||
times = [parse_iso_timestamp(s) for s in starred]
|
||||
base = sum(1 for t in times if t < start)
|
||||
times = [t for t in times if t >= start]
|
||||
# Anchor the line at the start date so the curve begins at the axis edge.
|
||||
x = [mdates.date2num(start)] + [mdates.date2num(t) for t in times]
|
||||
y = [base] + [base + i for i in range(1, len(times) + 1)]
|
||||
return np.array(x), np.array(y)
|
||||
|
||||
|
||||
def pick_xticks(x0: float, x1: float) -> tuple[list[float], str]:
|
||||
"""Evenly spaced x tick positions plus a date format for the given span.
|
||||
|
||||
Ticks are anchored at the newest date and step backwards, so the latest
|
||||
day is always labeled. The granularity coarsens from days to months to
|
||||
years as the history grows, keeping the label count at or below
|
||||
MAX_XTICKS instead of drawing one tick per day forever.
|
||||
"""
|
||||
start = mdates.num2date(x0)
|
||||
end = mdates.num2date(x1)
|
||||
span_days = x1 - x0
|
||||
|
||||
for step in DAY_STEPS:
|
||||
if span_days / step <= MAX_XTICKS:
|
||||
anchor = end.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
ticks = []
|
||||
while (num := mdates.date2num(anchor)) >= x0:
|
||||
ticks.append(num)
|
||||
anchor -= timedelta(days=step)
|
||||
fmt = "%b %-d" if start.year == end.year else "%b %-d, %Y"
|
||||
return sorted(ticks), fmt
|
||||
|
||||
span_months = (end.year - start.year) * 12 + end.month - start.month
|
||||
for step in MONTH_STEPS:
|
||||
if span_months / step <= MAX_XTICKS:
|
||||
# Month starts read better than an offset from "today" here.
|
||||
year, month = end.year, end.month
|
||||
ticks = []
|
||||
while (num := mdates.date2num(end.replace(
|
||||
year=year, month=month, day=1, hour=0, minute=0, second=0, microsecond=0
|
||||
))) >= x0:
|
||||
ticks.append(num)
|
||||
month -= step
|
||||
while month < 1:
|
||||
month += 12
|
||||
year -= 1
|
||||
fmt = "%b %Y" if start.year != end.year else "%b"
|
||||
return sorted(ticks), fmt
|
||||
|
||||
# Year granularity is the coarsest fallback, so widen the step as far as
|
||||
# needed rather than giving up and returning a crowded axis.
|
||||
span_years = end.year - start.year
|
||||
step = next(
|
||||
(s for s in YEAR_STEPS if span_years / s <= MAX_XTICKS),
|
||||
max(1, -(-span_years // MAX_XTICKS)),
|
||||
)
|
||||
year = end.year
|
||||
ticks = []
|
||||
while (num := mdates.date2num(end.replace(
|
||||
year=year, month=1, day=1, hour=0, minute=0, second=0, microsecond=0
|
||||
))) >= x0:
|
||||
ticks.append(num)
|
||||
year -= step
|
||||
return sorted(ticks), "%Y"
|
||||
|
||||
|
||||
def thin_xticklabels(fig, ax, min_gap: float = 14.0) -> None:
|
||||
"""Drop every n-th label until neighbours no longer crowd each other.
|
||||
|
||||
pick_xticks bounds the tick *count*, but whether the labels actually fit
|
||||
depends on the rendered text width and figure size, so measure the drawn
|
||||
labels and thin from the right (keeping the newest date) until every pair
|
||||
is at least `min_gap` pixels apart.
|
||||
"""
|
||||
ticks = list(ax.get_xticks())
|
||||
for keep in range(1, max(len(ticks), 1) + 1):
|
||||
kept = ticks[::-1][::keep][::-1]
|
||||
ax.set_xticks(kept)
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
boxes = [
|
||||
lbl.get_window_extent(renderer=renderer)
|
||||
for lbl in ax.get_xticklabels()
|
||||
if lbl.get_text()
|
||||
]
|
||||
if all(
|
||||
nxt.x0 - cur.x1 >= min_gap for cur, nxt in zip(boxes, boxes[1:])
|
||||
):
|
||||
return
|
||||
|
||||
|
||||
def draw(x: np.ndarray, y: np.ndarray, repo: str, theme_name: str, theme: dict, out: Path) -> None:
|
||||
bg, text, subtext, grid = theme["bg"], theme["text"], theme["subtext"], theme["grid"]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(12, 6.2), dpi=200)
|
||||
fig.patch.set_facecolor(bg)
|
||||
ax.set_facecolor(bg)
|
||||
fig.subplots_adjust(left=0.075, right=0.97, top=0.80, bottom=0.10)
|
||||
|
||||
ax.set_ylim(0, y.max() * 1.10)
|
||||
ax.set_xlim(x[0], x[-1] + (x[-1] - x[0]) * 0.03)
|
||||
|
||||
# Gradient fill under the curve: accent fading from top to transparent.
|
||||
r, g, b, _ = to_rgba(ACCENT)
|
||||
fade = LinearSegmentedColormap.from_list("fade", [(r, g, b, 0.0), (r, g, b, 0.35)])
|
||||
grad = np.linspace(0, 1, 256).reshape(-1, 1)
|
||||
im = ax.imshow(
|
||||
grad,
|
||||
aspect="auto",
|
||||
cmap=fade,
|
||||
origin="lower",
|
||||
extent=[ax.get_xlim()[0], ax.get_xlim()[1], 0, ax.get_ylim()[1]],
|
||||
zorder=1,
|
||||
)
|
||||
xs = np.concatenate([[x[0]], x, [x[-1]]])
|
||||
ys = np.concatenate([[0.0], y, [0.0]])
|
||||
(clip,) = ax.fill(xs, ys, alpha=0, zorder=1)
|
||||
im.set_clip_path(clip)
|
||||
|
||||
# Glow underlay + main line.
|
||||
ax.plot(x, y, color=ACCENT, linewidth=7, alpha=0.10, solid_capstyle="round", zorder=2)
|
||||
ax.plot(x, y, color=ACCENT, linewidth=2.6, solid_capstyle="round", zorder=3)
|
||||
|
||||
# Latest value: end dot + bold annotation.
|
||||
ax.scatter([x[-1]], [y[-1]], s=70, color=ACCENT, edgecolor=bg, linewidth=2.2, zorder=4)
|
||||
ax.annotate(
|
||||
f"{int(y[-1]):,} stars",
|
||||
xy=(x[-1], y[-1]),
|
||||
xytext=(-6, 14),
|
||||
textcoords="offset points",
|
||||
ha="right",
|
||||
fontsize=16,
|
||||
fontweight="bold",
|
||||
color=text,
|
||||
)
|
||||
|
||||
# Titles.
|
||||
fig.text(0.075, 0.93, "Star History", fontsize=22, fontweight="bold", color=text)
|
||||
fig.text(0.075, 0.862, repo, fontsize=12.5, color=subtext)
|
||||
|
||||
# Grid, spines, ticks.
|
||||
ax.yaxis.grid(True, color=grid, linewidth=0.9, linestyle=(0, (5, 4)))
|
||||
ax.set_axisbelow(True)
|
||||
for side in ("top", "right", "left"):
|
||||
ax.spines[side].set_visible(False)
|
||||
ax.spines["bottom"].set_color(grid)
|
||||
ax.tick_params(axis="both", length=0, labelsize=11.5, colors=subtext, pad=8)
|
||||
ticks, date_fmt = pick_xticks(*ax.get_xlim())
|
||||
ax.set_xticks(ticks)
|
||||
ax.xaxis.set_major_formatter(mdates.DateFormatter(date_fmt))
|
||||
ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _pos: f"{int(v):,}"))
|
||||
thin_xticklabels(fig, ax)
|
||||
|
||||
fig.savefig(out, facecolor=bg, bbox_inches="tight", pad_inches=0.3)
|
||||
plt.close(fig)
|
||||
print(f"wrote {out}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--repo", default=REPO)
|
||||
parser.add_argument("--start-date", default=START_DATE)
|
||||
parser.add_argument("--out-dir", default="assets")
|
||||
parser.add_argument("--refresh", action="store_true", help="ignore the timestamp cache")
|
||||
args = parser.parse_args()
|
||||
|
||||
start = parse_iso_timestamp(args.start_date)
|
||||
starred = fetch_starred_at(args.repo, refresh=args.refresh)
|
||||
x, y = build_series(starred, start)
|
||||
|
||||
out_dir = Path(args.out_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for name, theme in THEMES.items():
|
||||
draw(x, y, args.repo, name, theme, out_dir / f"star-history-{name}.png")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Use tracked source files for revision dates in the assembled MkDocs site.
|
||||
|
||||
``scripts/build_site.sh`` copies documentation into the ignored ``_web/``
|
||||
directory before MkDocs runs. The git revision-date plugin would otherwise
|
||||
query those generated paths, find no history, and give every page the build
|
||||
time. This hook primes the plugin's timestamp cache with the corresponding
|
||||
tracked source paths before the plugin's own ``on_files`` handler runs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from typing import Any, Iterable
|
||||
|
||||
from mkdocs.plugins import event_priority
|
||||
|
||||
from site_source_paths import REPO_ROOT, git_commit_range, original_source_map
|
||||
|
||||
_PLUGIN_NAME = "git-revision-date-localized"
|
||||
|
||||
|
||||
@event_priority(100)
|
||||
def on_files(files: Iterable[Any], config: Any, **_: Any) -> None:
|
||||
"""Populate the revision plugin's caches before its default-priority hook."""
|
||||
plugin = config.plugins.get(_PLUGIN_NAME)
|
||||
if plugin is None:
|
||||
return
|
||||
|
||||
sources = original_source_map(files)
|
||||
tracked = _tracked_paths()
|
||||
jobs = [
|
||||
(staged, source)
|
||||
for staged, source in sources.items()
|
||||
if _relative_source(source) in tracked
|
||||
]
|
||||
|
||||
ignored = tuple(getattr(plugin.util, "ignored_commits", ()))
|
||||
follow = bool(plugin.config.get("enable_git_follow"))
|
||||
include_creation = bool(plugin.config.get("enable_creation_date"))
|
||||
|
||||
def read_dates(job: tuple[str, str]):
|
||||
staged, source = job
|
||||
dates = git_commit_range(
|
||||
Path(source),
|
||||
ignored_commits=ignored,
|
||||
follow=follow,
|
||||
include_creation=include_creation,
|
||||
)
|
||||
return staged, dates
|
||||
|
||||
plugin.last_revision_commits.clear()
|
||||
plugin.created_commits.clear()
|
||||
|
||||
# Cache under the staged path because that is the key the plugin looks up
|
||||
# while rendering. Doing this at priority 100 also makes its default
|
||||
# on_files handler see a populated cache and skip querying `_web/` itself.
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
for staged, (latest, created) in executor.map(read_dates, jobs):
|
||||
plugin.last_revision_commits[staged] = latest
|
||||
if include_creation:
|
||||
plugin.created_commits[staged] = created
|
||||
|
||||
|
||||
def _relative_source(source: str) -> str:
|
||||
return Path(source).resolve().relative_to(REPO_ROOT).as_posix()
|
||||
|
||||
|
||||
def _tracked_paths() -> set[str]:
|
||||
output = subprocess.run(
|
||||
["git", "-C", str(REPO_ROOT), "ls-files", "-z"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
).stdout
|
||||
return {path.decode() for path in output.split(b"\0") if path}
|
||||
@@ -0,0 +1,36 @@
|
||||
"""MkDocs hook: strip Pandoc-specific attributes before rendering.
|
||||
|
||||
The book source uses Pandoc/LaTeX attributes that Python-Markdown does not
|
||||
understand and would otherwise render as literal text:
|
||||
|
||||
## 标题 {.unnumbered} -> ## 标题
|
||||
{height=55%} -> 
|
||||
[文本](#sec:foo){.unnumbered} -> [文本](#sec:foo)
|
||||
"""
|
||||
import re
|
||||
|
||||
_CODE_PATTERN = re.compile(r"(?P<fence>```+|~~~+|`+)([\s\S]*?)(?P=fence)")
|
||||
_PANDOC_ATTR = re.compile(
|
||||
r"[ \t]*\{(?:\s*#[a-zA-Z0-9_.:-]+|\s*\.[a-zA-Z0-9_-]+|\s*[a-zA-Z0-9_-]+=[^{}]*)+\s*\}"
|
||||
)
|
||||
|
||||
|
||||
def on_page_markdown(markdown, **kwargs):
|
||||
"""MkDocs hook to strip Pandoc attributes outside code blocks and inline code."""
|
||||
if not markdown:
|
||||
return ""
|
||||
out = []
|
||||
last_end = 0
|
||||
for match in _CODE_PATTERN.finditer(markdown):
|
||||
start, end = match.span()
|
||||
if start > last_end:
|
||||
non_code = markdown[last_end:start]
|
||||
non_code = _PANDOC_ATTR.sub("", non_code)
|
||||
out.append(non_code)
|
||||
out.append(match.group(0))
|
||||
last_end = end
|
||||
if last_end < len(markdown):
|
||||
non_code = markdown[last_end:]
|
||||
non_code = _PANDOC_ATTR.sub("", non_code)
|
||||
out.append(non_code)
|
||||
return "".join(out)
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Inject Open Graph + Twitter Card meta tags into every page so links
|
||||
look rich when shared to WeChat / Twitter / Slack.
|
||||
|
||||
Material's `social` plugin can do this but needs cairosvg + image
|
||||
rendering that silently no-ops in some CI environments. This hook is
|
||||
simpler: it derives all tags from page + site config, no images.
|
||||
|
||||
og:image points at assets/og-card.png — a pre-generated 1200x630 card
|
||||
checked into the repo (regenerate with scripts/gen_og_card.py when the
|
||||
branding text changes). One default card for every page.
|
||||
"""
|
||||
import html
|
||||
|
||||
|
||||
def on_post_page(output, page, config, **kwargs):
|
||||
meta = page.meta or {}
|
||||
title = meta.get("title") or page.title or config.get("site_name", "")
|
||||
desc = meta.get("description") or config.get("site_description", "")
|
||||
url = page.canonical_url or ""
|
||||
site = config.get("site_name", "")
|
||||
|
||||
# Static share-card image, generated by scripts/gen_og_card.py and
|
||||
# checked in as assets/og-card.png (1200x630). One default card for all
|
||||
# pages — crawlers need an absolute URL and a raster format, so the SVG
|
||||
# logo won't do.
|
||||
image = ""
|
||||
site_url = (config.get("site_url") or "").rstrip("/")
|
||||
if site_url:
|
||||
image = f"{site_url}/assets/og-card.png"
|
||||
|
||||
def add(attr, key, val):
|
||||
return f'<meta {attr}="{key}" content="{html.escape(str(val), quote=True)}">'
|
||||
|
||||
tags = [
|
||||
add("property", "og:type", "website"),
|
||||
add("property", "og:site_name", site),
|
||||
add("property", "og:title", title),
|
||||
add("property", "og:description", desc),
|
||||
add("property", "og:url", url),
|
||||
add("property", "og:locale", "zh_CN"),
|
||||
add("property", "og:image", image),
|
||||
add("property", "og:image:width", "1200" if image else ""),
|
||||
add("property", "og:image:height", "630" if image else ""),
|
||||
add("name", "twitter:card", "summary_large_image" if image else "summary"),
|
||||
add("name", "twitter:title", title),
|
||||
add("name", "twitter:description", desc),
|
||||
add("name", "twitter:image", image),
|
||||
]
|
||||
block = "\n".join(t for t in tags if 'content=""' not in t)
|
||||
return output.replace("</head>", block + "\n</head>", 1)
|
||||
@@ -0,0 +1,347 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate and build the static site's browser-side translation catalog.
|
||||
|
||||
The site contains all book editions in one MkDocs build. MkDocs Material can
|
||||
only use one ``theme.language`` per build, so its generated chrome is Chinese
|
||||
and translated editions localize it in the browser. This hook combines:
|
||||
|
||||
* Material for MkDocs' own locale catalogs (search, actions, footer, etc.); and
|
||||
* ``extras/site-nav-i18n.json`` (book navigation and custom controls).
|
||||
|
||||
Run ``python scripts/site_i18n.py`` after installing ``requirements-docs.txt``
|
||||
to audit every language. During ``mkdocs build`` the same validation runs and
|
||||
the hook emits ``_web/extras/site-i18n.generated.js`` for the browser.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
MKDOCS_CONFIG = ROOT / "mkdocs.yml"
|
||||
CUSTOM_CATALOG = ROOT / "extras" / "site-nav-i18n.json"
|
||||
GENERATED_CATALOG = ROOT / "_web" / "extras" / "site-i18n.generated.js"
|
||||
|
||||
# Material strings that the configured theme features can render. Search
|
||||
# result strings are updated dynamically, while the rest occur in the initial
|
||||
# document or in tooltips/dialogs.
|
||||
REQUIRED_UI_KEYS = (
|
||||
"action.edit",
|
||||
"action.skip",
|
||||
"action.view",
|
||||
"clipboard.copy",
|
||||
"clipboard.copied",
|
||||
"footer",
|
||||
"footer.next",
|
||||
"footer.previous",
|
||||
"header",
|
||||
"nav",
|
||||
"search",
|
||||
"search.placeholder",
|
||||
"search.share",
|
||||
"search.reset",
|
||||
"search.result.initializer",
|
||||
"search.result.placeholder",
|
||||
"search.result.none",
|
||||
"search.result.one",
|
||||
"search.result.other",
|
||||
"search.result.more.one",
|
||||
"search.result.more.other",
|
||||
"search.result.term.missing",
|
||||
"select.language",
|
||||
"source",
|
||||
"source.file.contributors",
|
||||
"source.file.date.created",
|
||||
"source.file.date.updated",
|
||||
"tabs",
|
||||
"toc",
|
||||
"top",
|
||||
)
|
||||
|
||||
CUSTOM_GROUPS = {
|
||||
"sidebar": ("show", "hide"),
|
||||
"palette": ("light", "dark"),
|
||||
}
|
||||
|
||||
HAN_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff]")
|
||||
|
||||
|
||||
class CatalogError(RuntimeError):
|
||||
"""Raised when the checked-in translation catalog is incomplete."""
|
||||
|
||||
|
||||
def configured_languages(config_text: str) -> dict[str, dict[str, str]]:
|
||||
"""Read inline ``extra.languages`` entries without a YAML dependency."""
|
||||
match = re.search(r"(?ms)^ languages:\s*\n(?P<body>.*?)(?=^nav:\s*$)", config_text)
|
||||
if not match:
|
||||
raise CatalogError("mkdocs.yml: could not find extra.languages")
|
||||
languages: dict[str, dict[str, str]] = {}
|
||||
for code, attributes in re.findall(
|
||||
r"(?m)^ ([a-zA-Z][a-zA-Z0-9_-]*):\s*\{([^}]*)\}\s*$",
|
||||
match.group("body"),
|
||||
):
|
||||
parsed: dict[str, str] = {}
|
||||
for key in ("prefix", "suffix", "readmeSuffix"):
|
||||
value = re.search(rf"(?:^|,)\s*{key}:\s*([^,]+)", attributes)
|
||||
if value:
|
||||
parsed[key] = value.group(1).strip().strip("\"'")
|
||||
languages[code] = parsed
|
||||
if not languages:
|
||||
raise CatalogError("mkdocs.yml: no inline extra.languages entries found")
|
||||
return languages
|
||||
|
||||
|
||||
def canonical_nav_labels(config_text: str) -> list[str]:
|
||||
"""Discover the named entries in the canonical MkDocs nav tree."""
|
||||
match = re.search(r"(?ms)^nav:\s*\n(?P<body>.*)$", config_text)
|
||||
if not match:
|
||||
raise CatalogError("mkdocs.yml: could not find nav")
|
||||
|
||||
labels: list[str] = []
|
||||
pattern = re.compile(r'''(?m)^\s*-\s+(?:"((?:[^"\\]|\\.)*)"|'((?:[^'\\]|''|\\.)*)'|([^:\n]+)):(?:\s|$)''')
|
||||
for double_q, single_q, unquoted in pattern.findall(match.group("body")):
|
||||
if double_q:
|
||||
label = double_q.replace(r'\"', '"').replace(r'\\', '\\').strip()
|
||||
elif single_q:
|
||||
label = single_q.replace(r"\'", "'").replace("''", "'").replace(r'\\', '\\').strip()
|
||||
else:
|
||||
label = (unquoted or "").strip().strip("\"'")
|
||||
if label and label not in labels:
|
||||
labels.append(label)
|
||||
if not labels:
|
||||
raise CatalogError("mkdocs.yml: no named nav entries found")
|
||||
return labels
|
||||
|
||||
|
||||
def material_languages_dir() -> Path:
|
||||
try:
|
||||
import material
|
||||
except ImportError as exc: # pragma: no cover - depends on caller's env
|
||||
raise CatalogError(
|
||||
"mkdocs-material is required; install requirements-docs.txt first"
|
||||
) from exc
|
||||
return Path(material.__file__).resolve().parent / "templates" / "partials" / "languages"
|
||||
|
||||
|
||||
def load_material_locale(locale: str, languages_dir: Path) -> dict[str, str]:
|
||||
path = languages_dir / f"{locale}.html"
|
||||
if not path.is_file():
|
||||
raise CatalogError(f"Material locale does not exist: {path}")
|
||||
|
||||
text = path.read_text(encoding="utf-8")
|
||||
start_match = re.search(r'\{\s*\n\s*"language"\s*:', text)
|
||||
if not start_match:
|
||||
raise CatalogError(f"Could not parse Material locale: {path}")
|
||||
end = text.find("}[key]", start_match.start())
|
||||
if end < 0:
|
||||
raise CatalogError(f"Could not parse Material locale: {path}")
|
||||
try:
|
||||
values = ast.literal_eval(text[start_match.start() : end + 1])
|
||||
except (SyntaxError, ValueError) as exc:
|
||||
raise CatalogError(f"Could not parse Material locale: {path}: {exc}") from exc
|
||||
if not isinstance(values, dict):
|
||||
raise CatalogError(f"Material locale is not a mapping: {path}")
|
||||
return values
|
||||
|
||||
|
||||
def load_custom_catalog() -> dict[str, dict[str, Any]]:
|
||||
try:
|
||||
data = json.loads(CUSTOM_CATALOG.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise CatalogError(f"Could not read {CUSTOM_CATALOG}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise CatalogError(f"{CUSTOM_CATALOG}: top level must be an object")
|
||||
return data
|
||||
|
||||
|
||||
def _check_nonempty(errors: list[str], code: str, field: str, value: Any) -> None:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
errors.append(f"{code}: {field} must be a non-empty string")
|
||||
|
||||
|
||||
def build_catalog() -> dict[str, Any]:
|
||||
"""Validate all sources and return the compact browser catalog."""
|
||||
config_text = MKDOCS_CONFIG.read_text(encoding="utf-8")
|
||||
configured = configured_languages(config_text)
|
||||
codes = list(configured)
|
||||
nav_labels = canonical_nav_labels(config_text)
|
||||
custom = load_custom_catalog()
|
||||
languages_dir = material_languages_dir()
|
||||
errors: list[str] = []
|
||||
|
||||
missing_codes = sorted(set(codes) - set(custom))
|
||||
extra_codes = sorted(set(custom) - set(codes))
|
||||
if missing_codes:
|
||||
errors.append(f"catalog is missing configured languages: {', '.join(missing_codes)}")
|
||||
if extra_codes:
|
||||
errors.append(f"catalog has unconfigured languages: {', '.join(extra_codes)}")
|
||||
|
||||
browser_languages: dict[str, Any] = {}
|
||||
for code in codes:
|
||||
entry = custom.get(code)
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
|
||||
material_locale = entry.get("material_locale")
|
||||
_check_nonempty(errors, code, "material_locale", material_locale)
|
||||
if not isinstance(material_locale, str) or not material_locale:
|
||||
continue
|
||||
try:
|
||||
material_ui = load_material_locale(material_locale, languages_dir)
|
||||
except CatalogError as exc:
|
||||
errors.append(str(exc))
|
||||
continue
|
||||
overrides = entry.get("ui_overrides", {})
|
||||
if not isinstance(overrides, dict):
|
||||
errors.append(f"{code}: ui_overrides must be an object")
|
||||
overrides = {}
|
||||
unknown_overrides = sorted(set(overrides) - set(material_ui))
|
||||
if unknown_overrides:
|
||||
errors.append(
|
||||
f"{code}: ui_overrides has unknown Material keys: "
|
||||
+ ", ".join(unknown_overrides)
|
||||
)
|
||||
effective_ui = {**material_ui, **overrides}
|
||||
|
||||
ui: dict[str, str] = {}
|
||||
for key in REQUIRED_UI_KEYS:
|
||||
value = effective_ui.get(key)
|
||||
_check_nonempty(errors, code, f"Material UI key {key}", value)
|
||||
if isinstance(value, str):
|
||||
ui[key] = value
|
||||
|
||||
nav = entry.get("nav")
|
||||
if not isinstance(nav, dict):
|
||||
errors.append(f"{code}: nav must be an object")
|
||||
nav = {}
|
||||
missing_nav = [label for label in nav_labels if label not in nav]
|
||||
extra_nav = [label for label in nav if label not in nav_labels]
|
||||
if missing_nav:
|
||||
errors.append(f"{code}: missing nav labels: {', '.join(missing_nav)}")
|
||||
if extra_nav:
|
||||
errors.append(f"{code}: unknown nav labels: {', '.join(extra_nav)}")
|
||||
for label in nav_labels:
|
||||
_check_nonempty(errors, code, f"nav.{label}", nav.get(label))
|
||||
|
||||
controls: dict[str, dict[str, str]] = {}
|
||||
for group, fields in CUSTOM_GROUPS.items():
|
||||
values = entry.get(group)
|
||||
if not isinstance(values, dict):
|
||||
errors.append(f"{code}: {group} must be an object")
|
||||
values = {}
|
||||
controls[group] = {}
|
||||
for field in fields:
|
||||
value = values.get(field)
|
||||
_check_nonempty(errors, code, f"{group}.{field}", value)
|
||||
if isinstance(value, str):
|
||||
controls[group][field] = value
|
||||
|
||||
# Chinese characters in a non-CJK catalog almost always mean a label
|
||||
# was copied but not translated. Japanese and Traditional Chinese are
|
||||
# excluded because Han characters are part of those target languages.
|
||||
if code not in {"zh", "zhtw", "ja"}:
|
||||
custom_values = list(nav.values()) + list(ui.values())
|
||||
for values in controls.values():
|
||||
custom_values.extend(values.values())
|
||||
leaked = [value for value in custom_values if isinstance(value, str) and HAN_RE.search(value)]
|
||||
if leaked:
|
||||
errors.append(f"{code}: Chinese text remains in custom UI: {leaked[0]!r}")
|
||||
|
||||
browser_languages[code] = {
|
||||
"locale": effective_ui.get("language", material_locale),
|
||||
"direction": effective_ui.get("direction", "ltr"),
|
||||
"nav": {label: nav[label] for label in nav_labels if label in nav},
|
||||
"ui": ui,
|
||||
**controls,
|
||||
}
|
||||
|
||||
# Every path produced by the language switcher must have a source
|
||||
# Markdown file. This catches naming drift such as a translated
|
||||
# reference-answer file retaining its old non-ASCII filename.
|
||||
language_config = configured[code]
|
||||
prefix = language_config.get("prefix")
|
||||
if not prefix:
|
||||
errors.append(f"{code}: mkdocs language config has no prefix")
|
||||
continue
|
||||
book_dir = ROOT / prefix.rstrip("/")
|
||||
suffix = language_config.get("suffix", "")
|
||||
prose_slugs = ["introduction", *(f"chapter{n}" for n in range(1, 11)), "afterword", "reference-answers"]
|
||||
for slug in prose_slugs:
|
||||
source = book_dir / f"{slug}{suffix}.md"
|
||||
if not source.is_file():
|
||||
errors.append(
|
||||
f"{code}: switcher URL has no source file: {source.relative_to(ROOT)}"
|
||||
)
|
||||
readme_suffix = language_config.get("readmeSuffix")
|
||||
if readme_suffix:
|
||||
for number in range(1, 11):
|
||||
source = ROOT / f"chapter{number}" / f"README.{readme_suffix}.md"
|
||||
if not source.is_file():
|
||||
errors.append(
|
||||
f"{code}: experiment-index URL has no source file: {source.relative_to(ROOT)}"
|
||||
)
|
||||
|
||||
if errors:
|
||||
raise CatalogError("Static-site i18n validation failed:\n - " + "\n - ".join(errors))
|
||||
|
||||
default = "zh"
|
||||
if default not in browser_languages:
|
||||
raise CatalogError(f"Default language {default!r} is not configured")
|
||||
|
||||
# Languages with a translated site homepage (root index.<code>.md).
|
||||
# The language switcher maps home <-> home for these editions and keeps
|
||||
# the introduction-page fallback for the rest. The default edition's
|
||||
# homepage is the root index.md itself.
|
||||
home_pages = [code for code in codes if (ROOT / f"index.{code}.md").is_file()]
|
||||
|
||||
return {
|
||||
"default": default,
|
||||
"languages": browser_languages,
|
||||
"canonicalNav": nav_labels,
|
||||
"homePages": home_pages,
|
||||
}
|
||||
|
||||
|
||||
def write_browser_catalog(catalog: dict[str, Any]) -> None:
|
||||
GENERATED_CATALOG.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = json.dumps(catalog, ensure_ascii=False, separators=(",", ":"))
|
||||
GENERATED_CATALOG.write_text(
|
||||
"// Generated by scripts/site_i18n.py; do not edit.\n"
|
||||
f"window.SITE_I18N={payload};\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def on_config(config: Any, **_: Any) -> Any:
|
||||
"""MkDocs hook: fail the build on drift and emit the browser catalog."""
|
||||
try:
|
||||
write_browser_catalog(build_catalog())
|
||||
except CatalogError as exc:
|
||||
from mkdocs.exceptions import ConfigurationError
|
||||
|
||||
raise ConfigurationError(str(exc)) from exc
|
||||
return config
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
catalog = build_catalog()
|
||||
except CatalogError as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(
|
||||
"Static-site i18n catalog is complete: "
|
||||
f"{len(catalog['languages'])} languages, "
|
||||
f"{len(catalog['canonicalNav'])} navigation labels each."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Map pages in the generated MkDocs tree to their repository sources."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path, PurePosixPath
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
Commit = tuple[str, int]
|
||||
|
||||
|
||||
def source_path_for_page(src_uri: str, root: Path = REPO_ROOT) -> Path:
|
||||
"""Return the tracked source represented by an assembled page URI."""
|
||||
relative = PurePosixPath(src_uri)
|
||||
parts = [p for p in relative.parts if p != "/"]
|
||||
|
||||
# build_site.sh promotes book/chapterN.md to book/chapterN/index.md so
|
||||
# navigation.indexes can make the chapter section itself clickable.
|
||||
if (
|
||||
len(parts) == 3
|
||||
and parts[0] == "book"
|
||||
and parts[1].startswith("chapter")
|
||||
and parts[1][7:].isdigit()
|
||||
and parts[2] == "index.md"
|
||||
):
|
||||
return root / "book" / f"{parts[1]}.md"
|
||||
|
||||
return root.joinpath(*parts)
|
||||
|
||||
|
||||
def original_source_map(files: Iterable[Any], root: Path = REPO_ROOT) -> dict[str, str]:
|
||||
"""Map staged absolute paths to existing source files in the repository."""
|
||||
sources: dict[str, str] = {}
|
||||
for file in files:
|
||||
abs_src_path = getattr(file, "abs_src_path", None)
|
||||
src_uri = getattr(file, "src_uri", None)
|
||||
if not abs_src_path or not src_uri:
|
||||
continue
|
||||
|
||||
source = source_path_for_page(str(src_uri), root)
|
||||
if source.is_file():
|
||||
sources[str(abs_src_path)] = str(source)
|
||||
return sources
|
||||
|
||||
|
||||
def git_commit_range(
|
||||
source: Path,
|
||||
root: Path = REPO_ROOT,
|
||||
*,
|
||||
ignored_commits: tuple[str, ...] = (),
|
||||
follow: bool = False,
|
||||
include_creation: bool = True,
|
||||
) -> tuple[Commit, Commit]:
|
||||
"""Return the latest and creation commits for one tracked source file."""
|
||||
relative = source.resolve().relative_to(root.resolve()).as_posix()
|
||||
common = ["git", "-C", str(root), "log", "--format=%H%x00%at"]
|
||||
if follow:
|
||||
common.append("--follow")
|
||||
|
||||
latest_lines = _git_log(common + [f"-n{len(ignored_commits) + 1}", "--", relative])
|
||||
latest = next(
|
||||
(
|
||||
commit
|
||||
for commit in map(_parse_commit, latest_lines)
|
||||
if not any(commit[0].startswith(prefix) for prefix in ignored_commits)
|
||||
),
|
||||
_fallback_commit(),
|
||||
)
|
||||
|
||||
if not include_creation:
|
||||
return latest, latest
|
||||
|
||||
creation_lines = _git_log(common + ["--diff-filter=A", "--", relative])
|
||||
valid_creation_commits = [
|
||||
commit
|
||||
for commit in map(_parse_commit, creation_lines)
|
||||
if not any(commit[0].startswith(prefix) for prefix in ignored_commits)
|
||||
]
|
||||
created = valid_creation_commits[-1] if valid_creation_commits else latest
|
||||
return latest, created
|
||||
|
||||
|
||||
def _git_log(command: list[str]) -> list[str]:
|
||||
output = subprocess.run(
|
||||
command,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout
|
||||
return [line for line in output.splitlines() if line]
|
||||
|
||||
|
||||
def _parse_commit(line: str) -> Commit:
|
||||
commit_hash, timestamp = line.split("\0", 1)
|
||||
return commit_hash, int(timestamp)
|
||||
|
||||
|
||||
def _fallback_commit() -> Commit:
|
||||
return "", int(time.time())
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Split Material's monolithic search index into one file per book edition.
|
||||
|
||||
MkDocs' search plugin emits a single ``search/search_index.json`` covering
|
||||
every page in the build. This site ships all 14 book editions plus the ~94
|
||||
companion-experiment pages from one build, so that file had grown to ~55 MB:
|
||||
every reader who opens search downloads the full prose of 13 editions they
|
||||
cannot read.
|
||||
|
||||
This hook rewrites the search plugin's output into:
|
||||
|
||||
* ``search/search_index.json`` — the default edition plus shared pages, kept
|
||||
at the canonical name so any client that does not run the router (or a
|
||||
stale cached page) still gets a working index; and
|
||||
* ``search/search_index.<slug>.json`` — one file per edition, where ``slug``
|
||||
is the edition's URL directory (``book``, ``book-en``, ``book-ta``, ...).
|
||||
|
||||
Every file also carries the *shared* pages — the language-agnostic experiment
|
||||
pages under ``chapterN/`` and the site root — so searching from any edition
|
||||
still reaches the companion experiments, exactly as it does today.
|
||||
|
||||
``extras/search-index-router.js`` selects the matching file in the browser.
|
||||
The two sides must agree on how a URL maps to an edition slug; see
|
||||
``edition_of()`` here and ``slugForPath()`` there.
|
||||
|
||||
Ordering: MkDocs appends ``hooks:`` entries to the plugin list
|
||||
(``config_options.Hooks.post_validation``), so this ``on_post_build`` runs
|
||||
after the search plugin has written the index it consumes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
log = logging.getLogger("mkdocs.hooks.split_search_index")
|
||||
|
||||
# `chapterN/README.<readmeSuffix>/` — the per-language experiment index pages.
|
||||
# They live outside the book-*/ tree but belong to a specific edition.
|
||||
README_RE = re.compile(r"^chapter\d+/README\.([A-Za-z-]+)/")
|
||||
# `index.<code>/` — translated homepages (e.g. index.ko.md -> index.ko/).
|
||||
HOMEPAGE_RE = re.compile(r"^index\.([A-Za-z-]+)/")
|
||||
|
||||
SHARED = "__shared__"
|
||||
|
||||
|
||||
def _edition_tables(config: Any) -> tuple[list[tuple[str, str]], dict[str, str]]:
|
||||
"""Return (prefix table, suffix->slug map) derived from `extra.languages`.
|
||||
|
||||
The prefix table is sorted longest-first so `book-en/` wins over `book/`
|
||||
when both would match.
|
||||
"""
|
||||
languages = (config.get("extra") or {}).get("languages") or {}
|
||||
|
||||
prefixes: list[tuple[str, str]] = []
|
||||
suffixes: dict[str, str] = {}
|
||||
for code, entry in languages.items():
|
||||
prefix = (entry or {}).get("prefix")
|
||||
if not prefix:
|
||||
continue
|
||||
slug = prefix.rstrip("/")
|
||||
prefixes.append((prefix, slug))
|
||||
# `readmeSuffix` keys the experiment index pages; the default edition
|
||||
# has none (its pages are `chapterN/README/`, which stay shared).
|
||||
readme_suffix = (entry or {}).get("readmeSuffix")
|
||||
if readme_suffix:
|
||||
suffixes[readme_suffix] = slug
|
||||
# Translated homepages are keyed by the language code itself.
|
||||
suffixes.setdefault(code, slug)
|
||||
|
||||
prefixes.sort(key=lambda pair: len(pair[0]), reverse=True)
|
||||
return prefixes, suffixes
|
||||
|
||||
|
||||
def edition_of(location: str, prefixes: list[tuple[str, str]], suffixes: dict[str, str]) -> str:
|
||||
"""Map a search-index location to an edition slug, or SHARED.
|
||||
|
||||
Mirrors `slugForPath()` in extras/search-index-router.js.
|
||||
"""
|
||||
for prefix, slug in prefixes:
|
||||
if location.startswith(prefix):
|
||||
return slug
|
||||
|
||||
match = README_RE.match(location) or HOMEPAGE_RE.match(location)
|
||||
if match:
|
||||
slug = suffixes.get(match.group(1))
|
||||
if slug:
|
||||
return slug
|
||||
|
||||
return SHARED
|
||||
|
||||
|
||||
def _default_slug(config: Any) -> str:
|
||||
languages = (config.get("extra") or {}).get("languages") or {}
|
||||
for entry in languages.values():
|
||||
if (entry or {}).get("default") and (entry or {}).get("prefix"):
|
||||
return entry["prefix"].rstrip("/")
|
||||
return "book"
|
||||
|
||||
|
||||
def on_post_build(config: Any, **_: Any) -> None:
|
||||
index_path = Path(config["site_dir"]) / "search" / "search_index.json"
|
||||
if not index_path.exists():
|
||||
# `search_index_only` themes or a disabled search plugin.
|
||||
log.debug("no search index at %s; nothing to split", index_path)
|
||||
return
|
||||
|
||||
data = json.loads(index_path.read_text(encoding="utf-8"))
|
||||
docs = data.get("docs")
|
||||
if not isinstance(docs, list):
|
||||
log.warning("unexpected search index shape; leaving it untouched")
|
||||
return
|
||||
|
||||
prefixes, suffixes = _edition_tables(config)
|
||||
if not prefixes:
|
||||
log.warning("no `extra.languages` prefixes; leaving the index untouched")
|
||||
return
|
||||
|
||||
buckets: dict[str, list[dict]] = {}
|
||||
for doc in docs:
|
||||
buckets.setdefault(edition_of(doc.get("location", ""), prefixes, suffixes), []).append(doc)
|
||||
|
||||
shared = buckets.pop(SHARED, [])
|
||||
if not buckets:
|
||||
log.warning("no edition pages found in the search index; leaving it untouched")
|
||||
return
|
||||
|
||||
def write(path: Path, entries: list[dict]) -> int:
|
||||
payload = dict(data)
|
||||
payload["docs"] = entries
|
||||
# `separators` matches what the search plugin emits; keeping the file
|
||||
# compact matters more here than diffability (it is build output).
|
||||
blob = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
path.write_text(blob, encoding="utf-8")
|
||||
return len(blob.encode("utf-8"))
|
||||
|
||||
total_before = index_path.stat().st_size
|
||||
written = []
|
||||
for slug, entries in sorted(buckets.items()):
|
||||
size = write(index_path.with_name(f"search_index.{slug}.json"), shared + entries)
|
||||
written.append((slug, len(entries), size))
|
||||
|
||||
# The canonical filename keeps serving the default edition, so a client
|
||||
# that never runs the router degrades to today's behaviour for that
|
||||
# edition instead of losing search entirely.
|
||||
default_slug = _default_slug(config)
|
||||
default_docs = buckets.get(default_slug, [])
|
||||
default_size = write(index_path, shared + default_docs)
|
||||
|
||||
log.info(
|
||||
"split search index: %.1f MB -> %d per-edition files of %.1f-%.1f MB "
|
||||
"(%d shared docs in each; default `%s` kept at search_index.json, %.1f MB)",
|
||||
total_before / 1e6,
|
||||
len(written),
|
||||
min(size for _, _, size in written) / 1e6,
|
||||
max(size for _, _, size in written) / 1e6,
|
||||
len(shared),
|
||||
default_slug,
|
||||
default_size / 1e6,
|
||||
)
|
||||
@@ -0,0 +1,947 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Synchronize localized Chapter 2 SVGs with the Chinese golden layouts.
|
||||
|
||||
The Chapter 2 figure sequence changed after several translations had copied an
|
||||
older set of diagrams. This script keeps the affected layouts tied to the
|
||||
Chinese edition while applying an explicit, reviewable localization map. It
|
||||
also applies the authoritative context-compression experiment measurements to
|
||||
Figures 2-16 and 2-17 in every edition.
|
||||
|
||||
Usage:
|
||||
python scripts/sync_chapter2_figures.py # all editions
|
||||
python scripts/sync_chapter2_figures.py --locale es
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import html
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
EDITIONS = {
|
||||
"zh": "book",
|
||||
"ar": "book-ar",
|
||||
"en": "book-en",
|
||||
"es": "book-es",
|
||||
"id": "book-id",
|
||||
"ja": "book-ja",
|
||||
"ko": "book-ko",
|
||||
"ru": "book-ru",
|
||||
"ta": "book-ta",
|
||||
"tr": "book-tr",
|
||||
"vi": "book-vi",
|
||||
"zhtw": "book-zhtw",
|
||||
}
|
||||
|
||||
# These editions inherited six diagrams from an obsolete Chapter 2 sequence.
|
||||
LAYOUT_SYNC_EDITIONS = {"ar", "en", "es", "id", "ja", "ru", "ta", "tr"}
|
||||
LAYOUT_SYNC_FIGURES = (2, 3, 4, 5, 8, 9)
|
||||
|
||||
# English is the complete fallback. Locale maps below override every piece of
|
||||
# prose while deliberately retaining API field names and special tokens.
|
||||
ENGLISH_TEXT = {
|
||||
2: [
|
||||
"Request (constructed by the agent framework)",
|
||||
"system",
|
||||
"Rules written by the developer",
|
||||
"user",
|
||||
'"Hello, who are you?"',
|
||||
"Call",
|
||||
"Response (returned by the API)",
|
||||
"assistant",
|
||||
"Model-generated reply",
|
||||
'"Hi! I\'m a coding assistant…"',
|
||||
"Each call is stateless — all information needed by the model must be fully provided in the request's messages list",
|
||||
],
|
||||
3: [
|
||||
"First call",
|
||||
"messages: system + user",
|
||||
"tools: get_current_time,",
|
||||
"get_weather",
|
||||
"API",
|
||||
"assistant: tool_calls",
|
||||
"get_current_time() +",
|
||||
"get_weather() (parallel)",
|
||||
"Agent framework executes two tools in parallel",
|
||||
"Second call",
|
||||
"messages: + tool results",
|
||||
"Vancouver time & weather",
|
||||
"Append to message history",
|
||||
"API",
|
||||
"assistant: final reply",
|
||||
"No tool call → end loop",
|
||||
'"Now it is…, and the weather is…"',
|
||||
"With a stateless API, the complete message history must be resent to the model in every round",
|
||||
],
|
||||
4: [
|
||||
"Static prefix (unchanged across rounds)",
|
||||
"System Prompt",
|
||||
"Tool Definitions",
|
||||
"Conversation history / trajectory (grows with interaction →)",
|
||||
"user",
|
||||
"assistant",
|
||||
"tool result",
|
||||
"user",
|
||||
"…",
|
||||
'"Static prefix + trajectory": keep the prefix fixed for KV Cache; the trajectory can be compressed',
|
||||
],
|
||||
5: [
|
||||
"User request",
|
||||
'"Help me contact Xfinity to negotiate"',
|
||||
"Local LLM service",
|
||||
"vLLM/Ollama (OpenAI compatible)",
|
||||
"Model inference",
|
||||
"Decide and generate tool_call",
|
||||
"Local tool execution",
|
||||
"Call function / external API",
|
||||
"Return tool results to the model, then generate the final response",
|
||||
],
|
||||
8: [
|
||||
"Structured API messages",
|
||||
"system",
|
||||
'"You are a helpful assistant."',
|
||||
"user",
|
||||
'"What is the weather in Beijing today?"',
|
||||
"assistant",
|
||||
"(to be generated)",
|
||||
"Chat Template",
|
||||
"Linear token stream actually processed by the model",
|
||||
"<|im_start|>system",
|
||||
"You are a helpful assistant.<|im_end|>",
|
||||
"<|im_start|>user",
|
||||
"What is the weather in Beijing today?<|im_end|>",
|
||||
"<|im_start|>assistant",
|
||||
"Special tokens mark roles and message boundaries, forming one continuous sequence",
|
||||
],
|
||||
9: [
|
||||
"API level (what developers see)",
|
||||
"{ ",
|
||||
'"role"',
|
||||
": ",
|
||||
'"system"',
|
||||
",",
|
||||
'"content"',
|
||||
": ",
|
||||
'"You are an assistant"',
|
||||
" }",
|
||||
"{ ",
|
||||
'"role"',
|
||||
": ",
|
||||
'"user"',
|
||||
",",
|
||||
'"content"',
|
||||
": ",
|
||||
'"Hello"',
|
||||
" }",
|
||||
"Model level (after Chat Template conversion)",
|
||||
"<|im_start|>",
|
||||
"system",
|
||||
"You are an assistant",
|
||||
"<|im_end|>",
|
||||
"<|im_start|>",
|
||||
"user",
|
||||
"Hello",
|
||||
"<|im_end|>",
|
||||
"<|im_start|>",
|
||||
"assistant",
|
||||
"(the model starts generating here)",
|
||||
],
|
||||
10: [
|
||||
"Request 1",
|
||||
"System Prompt + Tools (1200 tokens)",
|
||||
'user: "What is the weather?"',
|
||||
"→ Generate response",
|
||||
"Request 2",
|
||||
"System Prompt + Tools (cache hit ✓)",
|
||||
'user: "What time is it?"',
|
||||
"→ Generate response",
|
||||
"KV reuse",
|
||||
"Request 3",
|
||||
"(system prompt changed)",
|
||||
'System + Tools + "Time: 10:30:45"',
|
||||
'user: "What is the weather?"',
|
||||
"→ Full recomputation ✗",
|
||||
"Performance comparison (3000-token total context)",
|
||||
"Cache hit",
|
||||
"Cache miss",
|
||||
"TTFT",
|
||||
"~0.5 seconds",
|
||||
"3–5 seconds",
|
||||
"Cost",
|
||||
"Only new tokens billed",
|
||||
"All tokens billed again",
|
||||
],
|
||||
11: [
|
||||
"Layer 1: Metadata (loaded at startup, ~300 tokens)",
|
||||
'skills: [{name: "PPTX", desc: "Create PowerPoint presentations from content"}',
|
||||
' {name: "PDF", desc: "Extract and analyze PDF documents"}, ...]',
|
||||
'Task trigger: "Generate PPT from paper"',
|
||||
"Layer 2: SKILL.md core flow (loaded on demand, ~2K tokens)",
|
||||
"PPTX Skill core flow:",
|
||||
"1. markitdown extracts text → 2. Unzip PPTX to access XML",
|
||||
"3. Modify slide{N}.xml content → 4. Repackage as .pptx",
|
||||
"References: → html2pptx.md | → reference.md | → scripts/",
|
||||
'Need detailed method: "Create PPT with an HTML template"',
|
||||
"Layer 3: Subdocuments (selective deep dive, loaded on demand)",
|
||||
"html2pptx.md",
|
||||
"Complete workflow for",
|
||||
"HTML template → PPT",
|
||||
"reference.md",
|
||||
"XML format specification",
|
||||
"and technical details",
|
||||
"scripts/*.py",
|
||||
"Executable tools:",
|
||||
"thumbnail.py, etc.",
|
||||
"Fixed metadata → KV Cache friendly | Append dynamic content → keep cache valid",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
LOCALIZED_TEXT = {
|
||||
"ar": {
|
||||
2: [
|
||||
"الطلب (ينشئه إطار عمل الوكيل)", "system", "القواعد التي كتبها المطوّر", "user",
|
||||
'"مرحبًا، من أنت؟"', "استدعاء", "الاستجابة (تعيدها API)", "assistant",
|
||||
"رد أنشأه النموذج", '"مرحبًا! أنا مساعد برمجي…"',
|
||||
"كل استدعاء عديم الحالة — يجب توفير كل ما يحتاجه النموذج ضمن قائمة messages في الطلب",
|
||||
],
|
||||
3: [
|
||||
"الاستدعاء الأول", "messages: system + user", "tools: get_current_time,", "get_weather", "API",
|
||||
"assistant: tool_calls", "get_current_time() +", "get_weather() (بالتوازي)",
|
||||
"ينفّذ إطار عمل الوكيل الأداتين بالتوازي", "الاستدعاء الثاني", "messages: + نتائج الأدوات",
|
||||
"وقت فانكوفر والطقس", "إلحاق بسجل الرسائل", "API", "assistant: الرد النهائي",
|
||||
"لا استدعاء لأداة — إنهاء الحلقة", '"الوقت الآن…، والطقس…"',
|
||||
"مع API عديمة الحالة، يجب إعادة إرسال سجل الرسائل الكامل إلى النموذج في كل جولة",
|
||||
],
|
||||
4: [
|
||||
"بادئة ثابتة (لا تتغير بين الجولات)", "System Prompt (موجّه النظام)", "Tool Definitions (تعريفات الأدوات)",
|
||||
"سجل المحادثة / المسار (ينمو مع التفاعل ←)", "user", "assistant", "نتيجة الأداة", "user", "…",
|
||||
'بنية "البادئة الثابتة + المسار": تثبيت البادئة يفيد KV Cache، ويمكن ضغط المسار',
|
||||
],
|
||||
5: [
|
||||
"طلب المستخدم", '"ساعدني في التفاوض مع Xfinity"', "خدمة LLM محلية",
|
||||
"vLLM/Ollama (متوافقة مع OpenAI)", "استدلال النموذج", "تحديد tool_call وإنشاؤه",
|
||||
"تنفيذ الأدوات محليًا", "استدعاء دالة / API خارجية", "إعادة نتائج الأدوات إلى النموذج ثم إنشاء الرد النهائي",
|
||||
],
|
||||
8: [
|
||||
"رسائل API منظّمة", "system", '"أنت مساعد مفيد."', "user", '"كيف هو طقس بكين اليوم؟"',
|
||||
"assistant", "(في انتظار الإنشاء)", "Chat Template", "تدفق Token الخطي الذي يعالجه النموذج فعليًا",
|
||||
"<|im_start|>system", "أنت مساعد مفيد.<|im_end|>", "<|im_start|>user",
|
||||
"كيف هو طقس بكين اليوم؟<|im_end|>", "<|im_start|>assistant",
|
||||
"تحدد الرموز الخاصة الأدوار وحدود الرسائل لتكوين تسلسل متصل",
|
||||
],
|
||||
9: [
|
||||
"مستوى API (ما يراه المطوّر)", "{ ", '"role"', ": ", '"system"', ",", '"content"', ": ",
|
||||
'"أنت مساعد"', " }", "{ ", '"role"', ": ", '"user"', ",", '"content"', ": ", '"مرحبًا"', " }",
|
||||
"مستوى النموذج (بعد تحويل Chat Template)", "<|im_start|>", "system", "أنت مساعد", "<|im_end|>",
|
||||
"<|im_start|>", "user", "مرحبًا", "<|im_end|>", "<|im_start|>", "assistant",
|
||||
"(يبدأ النموذج الإنشاء من هنا)",
|
||||
],
|
||||
},
|
||||
"es": {
|
||||
2: [
|
||||
"Solicitud (construida por el framework del agente)", "system", "Reglas escritas por el desarrollador", "user",
|
||||
'"Hola, ¿quién eres?"', "Llamada", "Respuesta (devuelta por la API)", "assistant",
|
||||
"Respuesta generada por el modelo", '"¡Hola! Soy un asistente de programación…"',
|
||||
"Cada llamada no tiene estado: toda la información necesaria debe incluirse en la lista messages de la solicitud",
|
||||
],
|
||||
3: [
|
||||
"Primera llamada", "messages: system + user", "tools: get_current_time,", "get_weather", "API",
|
||||
"assistant: tool_calls", "get_current_time() +", "get_weather() (en paralelo)",
|
||||
"El framework del agente ejecuta dos herramientas en paralelo", "Segunda llamada",
|
||||
"messages: + resultados de herramientas", "Hora y tiempo de Vancouver", "Añadir al historial de mensajes",
|
||||
"API", "assistant: respuesta final", "Sin llamada a herramienta → fin del bucle",
|
||||
'"Ahora son las…, y el tiempo…"',
|
||||
"Con una API sin estado, hay que reenviar al modelo todo el historial en cada ronda",
|
||||
],
|
||||
4: [
|
||||
"Prefijo estático (no cambia entre rondas)", "System Prompt (prompt del sistema)",
|
||||
"Tool Definitions (definiciones de herramientas)", "Historial / trayectoria (crece con la interacción →)",
|
||||
"user", "assistant", "resultado de herramienta", "user", "…",
|
||||
'Estructura "prefijo estático + trayectoria": el prefijo se fija para KV Cache; la trayectoria se puede comprimir',
|
||||
],
|
||||
5: [
|
||||
"Solicitud del usuario", '"Ayúdame a negociar con Xfinity"', "Servicio LLM local",
|
||||
"vLLM/Ollama (compatible con OpenAI)", "Inferencia del modelo", "Decidir y generar tool_call",
|
||||
"Ejecución local de herramientas", "Llamar a función / API externa",
|
||||
"Devolver resultados al modelo y generar la respuesta final",
|
||||
],
|
||||
8: [
|
||||
"Mensajes estructurados de la API", "system", '"Eres un asistente útil."', "user",
|
||||
'"¿Qué tiempo hace hoy en Pekín?"', "assistant", "(pendiente de generar)", "Chat Template",
|
||||
"Flujo lineal de tokens que procesa realmente el modelo", "<|im_start|>system",
|
||||
"Eres un asistente útil.<|im_end|>", "<|im_start|>user",
|
||||
"¿Qué tiempo hace hoy en Pekín?<|im_end|>", "<|im_start|>assistant",
|
||||
"Los tokens especiales delimitan roles y mensajes para formar una secuencia continua",
|
||||
],
|
||||
9: [
|
||||
"Nivel de API (lo que ve el desarrollador)", "{ ", '"role"', ": ", '"system"', ",", '"content"', ": ",
|
||||
'"Eres un asistente"', " }", "{ ", '"role"', ": ", '"user"', ",", '"content"', ": ", '"Hola"', " }",
|
||||
"Nivel del modelo (tras convertir con Chat Template)", "<|im_start|>", "system", "Eres un asistente",
|
||||
"<|im_end|>", "<|im_start|>", "user", "Hola", "<|im_end|>", "<|im_start|>", "assistant",
|
||||
"(el modelo empieza a generar aquí)",
|
||||
],
|
||||
10: [
|
||||
"Solicitud 1", "System Prompt + Tools (1200 tokens)", 'user: "¿Qué tiempo hace?"', "→ Generar respuesta",
|
||||
"Solicitud 2", "System Prompt + Tools (acierto de caché ✓)", 'user: "¿Qué hora es?"', "→ Generar respuesta",
|
||||
"Reutilización de KV", "Sol. 3", "(prompt del sistema cambiado)",
|
||||
'System + Tools + "Time: 10:30:45"', 'user: "¿Qué tiempo hace?"', "→ Recalcular todo ✗",
|
||||
"Comparación de rendimiento (contexto total de 3000 tokens)", "Acierto de caché", "Fallo de caché",
|
||||
"TTFT", "~0,5 segundos", "3–5 segundos", "Coste", "Solo tokens nuevos",
|
||||
"Todos los tokens de nuevo",
|
||||
],
|
||||
11: [
|
||||
"Capa 1: Metadatos (cargados al inicio, ~300 tokens)",
|
||||
'skills: [{name: "PPTX", desc: "Crear presentaciones PowerPoint desde contenido"}',
|
||||
' {name: "PDF", desc: "Extraer y analizar documentos PDF"}, ...]',
|
||||
'Tarea activadora: "Generar PPT desde un artículo"',
|
||||
"Capa 2: Flujo principal de SKILL.md (bajo demanda, ~2K tokens)", "Flujo principal de PPTX Skill:",
|
||||
"1. markitdown extrae texto → 2. Descomprimir PPTX para acceder al XML",
|
||||
"3. Modificar slide{N}.xml → 4. Volver a empaquetar como .pptx",
|
||||
"Referencias: → html2pptx.md | → reference.md | → scripts/",
|
||||
'Método detallado: "Crear PPT con una plantilla HTML"',
|
||||
"Capa 3: Subdocumentos (consulta selectiva, bajo demanda)", "html2pptx.md", "Flujo completo para",
|
||||
"plantilla HTML → PPT", "reference.md", "Especificación del formato XML", "y detalles técnicos",
|
||||
"scripts/*.py", "Herramientas ejecutables:", "thumbnail.py, etc.",
|
||||
"Metadatos fijos → favorecen KV Cache | Contenido dinámico añadido → no invalida la caché",
|
||||
],
|
||||
},
|
||||
"id": {
|
||||
2: [
|
||||
"Request (disusun oleh framework Agent)", "system", "Aturan yang ditulis developer", "user",
|
||||
'"Halo, siapa kamu?"', "Panggil", "Response (dikembalikan API)", "assistant",
|
||||
"Jawaban yang dihasilkan model", '"Hai! Saya asisten pemrograman…"',
|
||||
"Setiap panggilan bersifat stateless — semua informasi harus lengkap dalam daftar messages pada request",
|
||||
],
|
||||
3: [
|
||||
"Panggilan pertama", "messages: system + user", "tools: get_current_time,", "get_weather", "API",
|
||||
"assistant: tool_calls", "get_current_time() +", "get_weather() (paralel)",
|
||||
"Framework Agent menjalankan dua tool secara paralel", "Panggilan kedua", "messages: + hasil tool",
|
||||
"Waktu & cuaca Vancouver", "Tambahkan ke riwayat pesan", "API", "assistant: jawaban akhir",
|
||||
"Tanpa panggilan tool → akhiri loop", '"Sekarang pukul…, cuacanya…"',
|
||||
"Pada API stateless, seluruh riwayat pesan harus dikirim ulang ke model di setiap putaran",
|
||||
],
|
||||
4: [
|
||||
"Prefix statis (tetap sama di setiap putaran)", "System Prompt", "Tool Definitions",
|
||||
"Riwayat percakapan / trajectory (terus bertambah →)", "user", "assistant", "hasil tool", "user", "…",
|
||||
'Struktur "prefix statis + trajectory": prefix dijaga tetap untuk KV Cache; trajectory dapat dikompresi',
|
||||
],
|
||||
5: [
|
||||
"Request pengguna", '"Bantu saya bernegosiasi dengan Xfinity"', "Layanan LLM lokal",
|
||||
"vLLM/Ollama (kompatibel dengan OpenAI)", "Inferensi model", "Tentukan dan hasilkan tool_call",
|
||||
"Eksekusi tool lokal", "Panggil fungsi / API eksternal", "Kembalikan hasil tool ke model lalu hasilkan jawaban akhir",
|
||||
],
|
||||
8: [
|
||||
"Pesan API terstruktur", "system", '"Anda adalah asisten yang membantu."', "user",
|
||||
'"Bagaimana cuaca Beijing hari ini?"', "assistant", "(belum dihasilkan)", "Chat Template",
|
||||
"Aliran Token linear yang benar-benar diproses model", "<|im_start|>system",
|
||||
"Anda adalah asisten yang membantu.<|im_end|>", "<|im_start|>user",
|
||||
"Bagaimana cuaca Beijing hari ini?<|im_end|>", "<|im_start|>assistant",
|
||||
"Token khusus menandai peran dan batas pesan, membentuk satu urutan kontinu",
|
||||
],
|
||||
9: [
|
||||
"Level API (yang dilihat developer)", "{ ", '"role"', ": ", '"system"', ",", '"content"', ": ",
|
||||
'"Anda adalah asisten"', " }", "{ ", '"role"', ": ", '"user"', ",", '"content"', ": ", '"Halo"', " }",
|
||||
"Level model (setelah konversi Chat Template)", "<|im_start|>", "system", "Anda adalah asisten",
|
||||
"<|im_end|>", "<|im_start|>", "user", "Halo", "<|im_end|>", "<|im_start|>", "assistant",
|
||||
"(model mulai menghasilkan dari sini)",
|
||||
],
|
||||
},
|
||||
"ja": {
|
||||
2: [
|
||||
"Request（Agent フレームワークが構築）", "system", "開発者が記述したルール", "user",
|
||||
'"こんにちは、あなたは誰ですか？"', "呼び出し", "Response（API が返却）", "assistant",
|
||||
"モデルが生成した応答", '"こんにちは！コーディングアシスタントです…"',
|
||||
"各呼び出しはステートレス — 必要な情報はすべて request の messages に含める",
|
||||
],
|
||||
3: [
|
||||
"1 回目の呼び出し", "messages: system + user", "tools: get_current_time,", "get_weather", "API",
|
||||
"assistant: tool_calls", "get_current_time() +", "get_weather()（並列）",
|
||||
"Agent フレームワークが 2 つの tool を並列実行", "2 回目の呼び出し", "messages: + tool の結果",
|
||||
"バンクーバーの時刻と天気", "メッセージ履歴に追加", "API", "assistant: 最終応答",
|
||||
"tool 呼び出しなし → ループ終了", '"現在は…、天気は…"',
|
||||
"ステートレス API では、毎回すべてのメッセージ履歴をモデルへ再送する",
|
||||
],
|
||||
4: [
|
||||
"静的プレフィックス（各ラウンドで不変）", "System Prompt（システムプロンプト）",
|
||||
"Tool Definitions（ツール定義）", "会話履歴 / 軌跡（対話とともに増加 →）", "user", "assistant",
|
||||
"tool の結果", "user", "…", "「静的プレフィックス + 軌跡」：KV Cache のためプレフィックスを固定し、軌跡は圧縮可能",
|
||||
],
|
||||
5: [
|
||||
"ユーザーの依頼", '"Xfinity との料金交渉を手伝って"', "ローカル LLM サービス",
|
||||
"vLLM/Ollama（OpenAI 互換）", "モデル推論", "tool_call を判断して生成",
|
||||
"ローカル tool 実行", "関数 / 外部 API を呼び出す", "tool の結果をモデルへ返し、最終応答を生成",
|
||||
],
|
||||
8: [
|
||||
"構造化された API メッセージ", "system", '"あなたは役に立つアシスタントです。"', "user",
|
||||
'"今日の北京の天気は？"', "assistant", "（生成待ち）", "Chat Template",
|
||||
"モデルが実際に処理する線形 Token ストリーム", "<|im_start|>system",
|
||||
"あなたは役に立つアシスタントです。<|im_end|>", "<|im_start|>user",
|
||||
"今日の北京の天気は？<|im_end|>", "<|im_start|>assistant",
|
||||
"特殊 Token が役割とメッセージ境界を示し、連続したシーケンスを形成",
|
||||
],
|
||||
9: [
|
||||
"API レベル（開発者から見える形式）", "{ ", '"role"', ": ", '"system"', ",", '"content"', ": ",
|
||||
'"あなたはアシスタントです"', " }", "{ ", '"role"', ": ", '"user"', ",", '"content"', ": ",
|
||||
'"こんにちは"', " }", "モデルレベル（Chat Template 変換後）", "<|im_start|>", "system",
|
||||
"あなたはアシスタントです", "<|im_end|>", "<|im_start|>", "user", "こんにちは", "<|im_end|>",
|
||||
"<|im_start|>", "assistant", "（モデルはここから生成を開始）",
|
||||
],
|
||||
},
|
||||
"ru": {
|
||||
2: [
|
||||
"Запрос (сформирован фреймворком агента)", "system", "Правила, заданные разработчиком", "user",
|
||||
'"Привет, кто ты?"', "Вызов", "Ответ (возвращён API)", "assistant", "Ответ, созданный моделью",
|
||||
'"Привет! Я ассистент по программированию…"',
|
||||
"Каждый вызов не хранит состояния — вся нужная информация должна быть в списке messages запроса",
|
||||
],
|
||||
3: [
|
||||
"Первый вызов", "messages: system + user", "tools: get_current_time,", "get_weather", "API",
|
||||
"assistant: tool_calls", "get_current_time() +", "get_weather() (параллельно)",
|
||||
"Фреймворк агента параллельно запускает два инструмента", "Второй вызов",
|
||||
"messages: + результаты инструментов", "Время и погода в Ванкувере", "Добавить в историю сообщений",
|
||||
"API", "assistant: итоговый ответ", "Нет вызова инструмента → завершить цикл",
|
||||
'"Сейчас…, погода…"', "При stateless API на каждом раунде модели повторно отправляется вся история сообщений",
|
||||
],
|
||||
4: [
|
||||
"Статический префикс (не меняется между раундами)", "System Prompt (системный промпт)",
|
||||
"Tool Definitions (описания инструментов)", "История диалога / траектория (постоянно растёт →)",
|
||||
"user", "assistant", "результат инструмента", "user", "…",
|
||||
'Структура «статический префикс + траектория»: префикс фиксирован для KV Cache, траекторию можно сжимать',
|
||||
],
|
||||
5: [
|
||||
"Запрос пользователя", '"Помоги договориться о скидке с Xfinity"', "Локальный сервис LLM",
|
||||
"vLLM/Ollama (совместим с OpenAI)", "Инференс модели", "Выбрать и создать tool_call",
|
||||
"Локальное выполнение инструмента", "Вызвать функцию / внешний API",
|
||||
"Вернуть результаты модели и сформировать итоговый ответ",
|
||||
],
|
||||
8: [
|
||||
"Структурированные сообщения API", "system", '"Ты полезный ассистент."', "user",
|
||||
'"Какая сегодня погода в Пекине?"', "assistant", "(ожидает генерации)", "Chat Template",
|
||||
"Линейный поток токенов, который фактически обрабатывает модель", "<|im_start|>system",
|
||||
"Ты полезный ассистент.<|im_end|>", "<|im_start|>user", "Какая сегодня погода в Пекине?<|im_end|>",
|
||||
"<|im_start|>assistant", "Специальные токены отмечают роли и границы сообщений, образуя непрерывную последовательность",
|
||||
],
|
||||
9: [
|
||||
"Уровень API (что видит разработчик)", "{ ", '"role"', ": ", '"system"', ",", '"content"', ": ",
|
||||
'"Ты ассистент"', " }", "{ ", '"role"', ": ", '"user"', ",", '"content"', ": ", '"Привет"', " }",
|
||||
"Уровень модели (после Chat Template)", "<|im_start|>", "system", "Ты ассистент", "<|im_end|>",
|
||||
"<|im_start|>", "user", "Привет", "<|im_end|>", "<|im_start|>", "assistant",
|
||||
"(модель начинает генерацию здесь)",
|
||||
],
|
||||
},
|
||||
"ta": {
|
||||
2: [
|
||||
"Request (Agent framework உருவாக்கியது)", "system", "Developer எழுதிய விதிகள்", "user",
|
||||
'"வணக்கம், நீங்கள் யார்?"', "அழைப்பு", "Response (API வழங்கியது)", "assistant",
|
||||
"Model உருவாக்கிய பதில்", '"வணக்கம்! நான் coding assistant…"',
|
||||
"ஒவ்வொரு அழைப்பும் stateless — தேவையான அனைத்தும் request-இன் messages பட்டியலில் முழுமையாக இருக்க வேண்டும்",
|
||||
],
|
||||
3: [
|
||||
"முதல் அழைப்பு", "messages: system + user", "tools: get_current_time,", "get_weather", "API",
|
||||
"assistant: tool_calls", "get_current_time() +", "get_weather() (இணையாக)",
|
||||
"Agent framework இரண்டு tools-ஐ இணையாக இயக்குகிறது", "இரண்டாம் அழைப்பு", "messages: + tool முடிவுகள்",
|
||||
"Vancouver நேரம் மற்றும் வானிலை", "Message history-இல் சேர்", "API", "assistant: இறுதிப் பதில்",
|
||||
"Tool call இல்லை → loop முடிவு", '"இப்போது…, வானிலை…"',
|
||||
"Stateless API-இல் ஒவ்வொரு சுற்றிலும் முழு message history-ஐ model-க்கு மீண்டும் அனுப்ப வேண்டும்",
|
||||
],
|
||||
4: [
|
||||
"நிலையான prefix (ஒவ்வொரு சுற்றிலும் மாறாது)", "System Prompt", "Tool Definitions",
|
||||
"உரையாடல் history / trajectory (தொடர்ந்து வளரும் →)", "user", "assistant", "tool முடிவு", "user", "…",
|
||||
'"நிலையான prefix + trajectory": KV Cache-க்காக prefix மாறாது; trajectory-ஐ compress செய்யலாம்',
|
||||
],
|
||||
5: [
|
||||
"பயனர் கோரிக்கை", '"Xfinity-யுடன் விலை பேச உதவுங்கள்"', "உள்ளூர் LLM சேவை",
|
||||
"vLLM/Ollama (OpenAI-compatible)", "Model inference", "tool_call-ஐ தீர்மானித்து உருவாக்கு",
|
||||
"உள்ளூர் tool இயக்கம்", "Function / வெளிப்புற API அழைப்பு", "Tool முடிவை model-க்கு அளித்து இறுதிப் பதிலை உருவாக்கு",
|
||||
],
|
||||
8: [
|
||||
"கட்டமைக்கப்பட்ட API messages", "system", '"நீங்கள் உதவிகரமான assistant."', "user",
|
||||
'"இன்று Beijing வானிலை எப்படி?"', "assistant", "(உருவாக்கப்பட வேண்டும்)", "Chat Template",
|
||||
"Model உண்மையில் செயலாக்கும் தொடர்ச்சியான Token stream", "<|im_start|>system",
|
||||
"நீங்கள் உதவிகரமான assistant.<|im_end|>", "<|im_start|>user",
|
||||
"இன்று Beijing வானிலை எப்படி?<|im_end|>", "<|im_start|>assistant",
|
||||
"Special tokens role மற்றும் message எல்லைகளைக் குறித்து ஒரே தொடரை உருவாக்குகின்றன",
|
||||
],
|
||||
9: [
|
||||
"API நிலை (developer காண்பது)", "{ ", '"role"', ": ", '"system"', ",", '"content"', ": ",
|
||||
'"நீங்கள் ஒரு assistant"', " }", "{ ", '"role"', ": ", '"user"', ",", '"content"', ": ", '"வணக்கம்"', " }",
|
||||
"Model நிலை (Chat Template மாற்றத்திற்குப் பின்)", "<|im_start|>", "system", "நீங்கள் ஒரு assistant",
|
||||
"<|im_end|>", "<|im_start|>", "user", "வணக்கம்", "<|im_end|>", "<|im_start|>", "assistant",
|
||||
"(model இங்கிருந்து உருவாக்கத் தொடங்குகிறது)",
|
||||
],
|
||||
},
|
||||
"tr": {
|
||||
2: [
|
||||
"Request (Agent framework tarafından oluşturulur)", "system", "Geliştiricinin yazdığı kurallar", "user",
|
||||
'"Merhaba, sen kimsin?"', "Çağrı", "Response (API tarafından döndürülür)", "assistant",
|
||||
"Modelin ürettiği yanıt", '"Merhaba! Ben bir kodlama asistanıyım…"',
|
||||
"Her çağrı stateless'tır — gereken tüm bilgiler request içindeki messages listesinde eksiksiz verilmelidir",
|
||||
],
|
||||
3: [
|
||||
"İlk çağrı", "messages: system + user", "tools: get_current_time,", "get_weather", "API",
|
||||
"assistant: tool_calls", "get_current_time() +", "get_weather() (paralel)",
|
||||
"Agent framework iki aracı paralel çalıştırır", "İkinci çağrı", "messages: + araç sonuçları",
|
||||
"Vancouver saati ve hava durumu", "Mesaj geçmişine ekle", "API", "assistant: son yanıt",
|
||||
"Araç çağrısı yok → döngüyü bitir", '"Şu an…, hava…"',
|
||||
"Stateless API'de tüm mesaj geçmişi her turda modele yeniden gönderilmelidir",
|
||||
],
|
||||
4: [
|
||||
"Statik önek (turlar boyunca değişmez)", "System Prompt (sistem istemi)",
|
||||
"Tool Definitions (araç tanımları)", "Konuşma geçmişi / trajectory (etkileşimle büyür →)",
|
||||
"user", "assistant", "araç sonucu", "user", "…",
|
||||
'"Statik önek + trajectory": KV Cache için önek sabit kalır; trajectory sıkıştırılabilir',
|
||||
],
|
||||
5: [
|
||||
"Kullanıcı isteği", '"Xfinity ile pazarlık yapmama yardım et"', "Yerel LLM hizmeti",
|
||||
"vLLM/Ollama (OpenAI uyumlu)", "Model çıkarımı", "tool_call seç ve oluştur",
|
||||
"Yerel araç yürütme", "Fonksiyon / harici API çağır", "Araç sonuçlarını modele verip son yanıtı oluştur",
|
||||
],
|
||||
8: [
|
||||
"Yapılandırılmış API mesajları", "system", '"Yardımcı bir asistansın."', "user",
|
||||
'"Pekin\'de bugün hava nasıl?"', "assistant", "(üretilecek)", "Chat Template",
|
||||
"Modelin gerçekte işlediği doğrusal Token akışı", "<|im_start|>system",
|
||||
"Yardımcı bir asistansın.<|im_end|>", "<|im_start|>user", "Pekin'de bugün hava nasıl?<|im_end|>",
|
||||
"<|im_start|>assistant", "Özel token'lar rol ve mesaj sınırlarını belirleyip kesintisiz bir dizi oluşturur",
|
||||
],
|
||||
9: [
|
||||
"API katmanı (geliştiricinin gördüğü)", "{ ", '"role"', ": ", '"system"', ",", '"content"', ": ",
|
||||
'"Sen bir asistansın"', " }", "{ ", '"role"', ": ", '"user"', ",", '"content"', ": ", '"Merhaba"', " }",
|
||||
"Model katmanı (Chat Template dönüşümünden sonra)", "<|im_start|>", "system", "Sen bir asistansın",
|
||||
"<|im_end|>", "<|im_start|>", "user", "Merhaba", "<|im_end|>", "<|im_start|>", "assistant",
|
||||
"(model burada üretmeye başlar)",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# Additional layout repairs found during the all-edition visual audit. These
|
||||
# maps intentionally use shorter labels where the golden geometry has narrow
|
||||
# columns; the meaning remains the same as the adjacent translated prose.
|
||||
LOCALIZED_TEXT.setdefault("ar", {}).update({
|
||||
14: [
|
||||
"بدون شريط حالة", "مع شريط الحالة", "النظام:", "موجّه النظام + الأدوات", "المستخدم:",
|
||||
'"تفاوض مع Xfinity"', "مساعد:", "phone_call(Xfinity) ← المحاولة 1", "الأداة:",
|
||||
"النتيجة: انتظار 45 د، لم يتصل", "مساعد:", 'web_search("عروض Xfinity")', "الأداة:",
|
||||
"النتيجة: [محتوى بحث كثير…]", "مساعد:", "phone_call(Xfinity) ← المحاولة 2", "الأداة:",
|
||||
"النتيجة: اتصال، عرض $65/شهر", "مساعد:", "phone_call(Xfinity) ← المحاولة 3", "الأداة:",
|
||||
"النتيجة: تأكيد $59/شهر", "المستخدم:", '"هل تتصل مجددًا؟"',
|
||||
"← يمسح النموذج السياق لعد المكالمات", "قد يخطئ في عددها", "النظام:",
|
||||
"موجّه النظام + الأدوات", "المستخدم:", '"تفاوض مع Xfinity"', "...:",
|
||||
"[محتوى المسار نفسه]", "المستخدم:", '"هل تتصل مجددًا؟"', "<agent_status>",
|
||||
"phone_call: 3 مرات (Xfinity: 3)", "حد المكالمات: بلغ (3/3) ✗",
|
||||
"TODO: [✓] اتصال [✓] تأكيد السعر", "الوقت: 2025-09-14 10:30",
|
||||
"الحالة: انتظار تأكيد المستخدم", "</agent_status>",
|
||||
"← يقرأ النموذج الحالة الموجزة مباشرة", "يلتزم بالحد ولا يجري مكالمات أخرى", "VS",
|
||||
],
|
||||
})
|
||||
|
||||
LOCALIZED_TEXT.setdefault("vi", {}).update({
|
||||
1: [
|
||||
"Từ nhắc hệ thống (System Prompt)",
|
||||
'"You are a helpful assistant. You MUST answer concisely."',
|
||||
'"Use tools when the user asks for real-time information."',
|
||||
"Định nghĩa tool (Tool Definitions)",
|
||||
'{"name": "web_search", "description": "Search the web",',
|
||||
'"parameters": {"query": {"type": "string"}}}',
|
||||
"Lịch sử hội thoại (Conversation History)",
|
||||
'user: "Thời tiết ở Bắc Kinh hôm nay thế nào?"',
|
||||
'assistant: [tool_call] → get_weather("Bắc Kinh")',
|
||||
'tool: {"temp": "23°C", "conditions": "trời quang"}',
|
||||
"Suy nghĩ trong lượt này (Reasoning)",
|
||||
"<think>Người dùng hỏi về thời tiết và tôi đã có kết quả từ tool,",
|
||||
"có thể tóm tắt và trả lời mà không cần gọi lại tool.</think>",
|
||||
"Vị trí sinh hiện tại →",
|
||||
'assistant: "Bắc Kinh hôm nay trời quang, 23°C…" ← LLM đang sinh',
|
||||
"Cửa sổ",
|
||||
"ngữ cảnh",
|
||||
"Kích thước cửa sổ: Qwen3 = 32K tokens | Claude = 200K | Gemini = 2M",
|
||||
"Toàn bộ nội dung được tuần tự thành luồng token → xử lý bởi attention của Transformer",
|
||||
],
|
||||
10: [
|
||||
"Yêu cầu 1", "System Prompt + Tools (1200 tokens)", 'user: "Thời tiết thế nào?"', "→ Tạo câu trả lời",
|
||||
"Yêu cầu 2", "System Prompt + Tools (cache hit ✓)", 'user: "Mấy giờ rồi?"', "→ Tạo câu trả lời",
|
||||
"Tái sử dụng KV", "YC 3", "(prompt đổi)", 'System + Tools + "Time: 10:30:45"',
|
||||
'user: "Thời tiết thế nào?"', "→ Tính lại toàn bộ ✗",
|
||||
"So sánh hiệu năng (tổng ngữ cảnh 3000 token)", "Cache trúng", "Cache trượt", "TTFT",
|
||||
"~0,5 giây", "3–5 giây", "Phí", "Chỉ tính token mới", "Tính lại toàn bộ token",
|
||||
],
|
||||
14: [
|
||||
"Không có thanh trạng thái", "Có thanh trạng thái", "system:", "System Prompt + Tools", "user:",
|
||||
'"Thương lượng giá với Xfinity"', "assistant:", "phone_call(Xfinity) → 1", "tool:",
|
||||
"KQ: chờ 45 phút, không kết nối", "assistant:", 'web_search("Xfinity deals")', "tool:",
|
||||
"KQ: [nhiều nội dung tìm kiếm…]", "assistant:", "phone_call(Xfinity) → 2", "tool:",
|
||||
"KQ: kết nối, báo giá $65/tháng", "assistant:", "phone_call(Xfinity) → 3", "tool:",
|
||||
"KQ: xác nhận giảm còn $59/tháng", "user:", '"Gọi lại để nhắc họ?"',
|
||||
"→ Mô hình quét ngữ cảnh để đếm số cuộc gọi", "Rất dễ đếm sai số cuộc gọi", "system:",
|
||||
"System Prompt + Tools", "user:", '"Thương lượng giá với Xfinity"', "...:",
|
||||
"[Cùng nội dung trajectory]", "user:", '"Gọi lại để nhắc họ?"', "<agent_status>",
|
||||
"phone_call: 3 lần (Xfinity: 3)", "Giới hạn: đã đạt (3/3) ✗",
|
||||
"TODO: [✓] Gọi Xfinity [✓] Xác nhận giá", "Thời gian: 2025-09-14 10:30",
|
||||
"Trạng thái: chờ người dùng xác nhận", "</agent_status>",
|
||||
"→ Mô hình đọc trực tiếp trạng thái cô đọng", "Tuân thủ giới hạn, không gọi thêm", "VS",
|
||||
],
|
||||
16: [
|
||||
"Chiến lược", "Token", "Tỷ lệ nén", "Số vòng", "Kết quả", "Trực quan (Token)",
|
||||
"Không nén", "166,043", "102.1%", "5", "✗ Thất bại",
|
||||
"Tóm tắt riêng lẻ", "276,608", "10.9%", "12", "✓ Thành công",
|
||||
"Tóm tắt tổng hợp", "93,449", "4.3%", "10", "✓ Thành công",
|
||||
"Theo ngữ cảnh", "40,157", "3.0%", "7", "✓ Thành công",
|
||||
"Có trích dẫn", "222,992", "4.1%", "10", "✓ Thành công",
|
||||
"Cửa sổ thích ứng", "174,601", "102.4%", "7", "✓ Thành công",
|
||||
"Nén theo ngữ cảnh: ít hơn 76% token so với không nén, đồng hạng ít vòng lặp nhất",
|
||||
"Điểm chính: đưa ý định truy vấn và thông tin hiện có vào quyết định nén",
|
||||
],
|
||||
})
|
||||
|
||||
LOCALIZED_TEXT.setdefault("id", {}).update({
|
||||
14: [
|
||||
"Tanpa status bar", "Dengan status bar", "system:", "System Prompt + Tools", "user:",
|
||||
'"Negosiasikan harga Xfinity"', "assistant:", "phone_call(Xfinity) → ke-1", "tool:",
|
||||
"Hasil: tunggu 45 mnt, tak tersambung", "assistant:", 'web_search("Promo Xfinity")', "tool:",
|
||||
"Hasil: [banyak hasil pencarian…]", "assistant:", "phone_call(Xfinity) → ke-2", "tool:",
|
||||
"Hasil: tersambung, tawaran $65/bln", "assistant:", "phone_call(Xfinity) → ke-3", "tool:",
|
||||
"Hasil: harga $59/bln dikonfirmasi", "user:", '"Telepon lagi untuk tindak lanjut?"',
|
||||
"→ Model memindai konteks untuk menghitung panggilan", "Rentan salah menghitung jumlah panggilan",
|
||||
"system:", "System Prompt + Tools", "user:", '"Negosiasikan harga Xfinity"', "...:",
|
||||
"[ Konten lintasan yang sama ]", "user:", '"Telepon lagi untuk tindak lanjut?"', "<agent_status>",
|
||||
"phone_call dipanggil 3 kali (Xfinity: 3)", "Cek batas: mencapai batas (3/3) ✗",
|
||||
"TODO: [✓]Hubungi [✓]Konfirmasi harga", "Waktu: 2025-09-14 10:30",
|
||||
"Status: menunggu konfirmasi pengguna", "</agent_status>",
|
||||
"→ Model langsung membaca status ringkas", "Patuh batasan, tidak ada panggilan lagi", "VS",
|
||||
],
|
||||
})
|
||||
|
||||
LOCALIZED_TEXT.setdefault("ta", {}).update({
|
||||
16: [
|
||||
"உத்தி", "Token", "விகிதம்", "சுற்று", "முடிவு", "காட்சி (Token)",
|
||||
"சுருக்கம் இல்லை", "166,043", "102.1%", "5", "✗ தோல்வி",
|
||||
"தனிப்பட்ட சுருக்கம்", "276,608", "10.9%", "12", "✓ வெற்றி",
|
||||
"ஒருங்கிணைந்த சுருக்கம்", "93,449", "4.3%", "10", "✓ வெற்றி",
|
||||
"சூழல்-உணர்வு", "40,157", "3.0%", "7", "✓ வெற்றி",
|
||||
"உணர்வு + மேற்கோள்", "222,992", "4.1%", "10", "✓ வெற்றி",
|
||||
"தகவமைப்பு சாளரம்", "174,601", "102.4%", "7", "✓ வெற்றி",
|
||||
"சூழல்-உணர்வு சுருக்கம்: சுருக்கமின்மையை விட 76% குறைந்த token; குறைந்த சுற்றுகளில் சமநிலை",
|
||||
"முக்கியம்: வினவல் நோக்கத்தையும் உள்ள தகவலையும் சுருக்க முடிவில் சேர்க்கவும்",
|
||||
],
|
||||
17: [
|
||||
"ஒவ்வொரு தேடலும் சராசரியாக ~52K எழுத்துகள் → ஒவ்வொரு உத்தியும் வேறுபடச் செயலாக்கும்",
|
||||
"① சுருக்கம் இல்லை", "நேரடியாக வைத்தல்", "முழு அசல் உரையை context-ல் வைத்தல்",
|
||||
"166K tok · 102.1% · தோல்வி", "② தனிப்பட்ட சுருக்கம்", "தனிச் சுருக்கம்",
|
||||
"ஒவ்வொரு முடிவுக்கும் தனியாக 2–3 பத்தி சுருக்கம்", "277K tok · 10.9% · 12 சுற்று",
|
||||
"③ ஒருங்கிணைந்த சுருக்கம்", "ஒன்றிணைந்த சுருக்கம்", "எல்லா முடிவுகளையும் இணைத்து ஒரே சுருக்கம்",
|
||||
"93K tok · 4.3% · 10 சுற்று", "④ சூழல்-உணர்வு", "நுண்ணறிவு சுருக்கம்",
|
||||
"Query + context → இலக்கு சுருக்கம்", "40K tok · 3.0% · 7 சுற்று",
|
||||
"⑤ உணர்வு + மேற்கோள்", "சுருக்கம் + மூலம்", "சுருக்கப்பட்ட உள்ளடக்கம் + URL மேற்கோள்கள்",
|
||||
"223K tok · 4.1% · 10 சுற்று", "⑥ தகவமைப்பு சாளரம்", "தாமத சுருக்கம்",
|
||||
"< 80% window-ல் அசல் உரை; மீறினால் batch compress", "175K tok · 102.4% · 7 சுற்று",
|
||||
],
|
||||
})
|
||||
|
||||
LOCALIZED_TEXT.setdefault("tr", {}).update({
|
||||
16: [
|
||||
"Strateji", "Token", "Oran", "Tur", "Sonuç", "Görsel (token kullanımı)",
|
||||
"Sıkıştırma yok", "166,043", "102.1%", "5", "✗ Başarısız",
|
||||
"Bireysel özet", "276,608", "10.9%", "12", "✓ Başarılı",
|
||||
"Birleşik özet", "93,449", "4.3%", "10", "✓ Başarılı",
|
||||
"Bağlam duyarlı", "40,157", "3.0%", "7", "✓ Başarılı",
|
||||
"Duyarlı + atıf", "222,992", "4.1%", "10", "✓ Başarılı",
|
||||
"Uyarlanır pencere", "174,601", "102.4%", "7", "✓ Başarılı",
|
||||
"Bağlam duyarlı sıkıştırma: sıkıştırmasız duruma göre %76 az token, en az turda eşit",
|
||||
"Anahtar: sorgu amacını ve mevcut bilgiyi sıkıştırma kararına katmak",
|
||||
],
|
||||
17: [
|
||||
"Her arama ortalama ~52K karakter döndürür → her strateji farklı biçimde işler",
|
||||
"① Sıkıştırma yok", "Doğrudan koru", "Özgün metnin tamamını bağlama ekle",
|
||||
"166K tok · %102,1 · başarısız", "② Bireysel özet", "Bağımsız özet",
|
||||
"Her sonuç için bağımsız 2–3 paragraflık özet", "277K tok · %10,9 · 12 tur",
|
||||
"③ Birleşik özet", "Birleşik özet", "Tüm sonuçları birleştirip tek özet oluştur",
|
||||
"93K tok · %4,3 · 10 tur", "④ Bağlam duyarlı", "Akıllı sıkıştırma",
|
||||
"Sorgu + bağlam → hedefli sıkıştırma", "40K tok · %3,0 · 7 tur",
|
||||
"⑤ Duyarlı + atıf", "Akıllı + izlenebilir", "Sıkıştırılmış içerik + URL atıf işaretleri",
|
||||
"223K tok · %4,1 · 10 tur", "⑥ Uyarlanır pencere", "Gecikmeli sıkıştırma",
|
||||
"< %80 pencerede özgün metin; aşınca toplu sıkıştır", "175K tok · %102,4 · 7 tur",
|
||||
],
|
||||
})
|
||||
|
||||
|
||||
FIG16_FOOTER = {
|
||||
"zh": "上下文感知压缩：相比无压缩节省 76% token，并列最少迭代次数",
|
||||
"ar": "الضغط المراعي للسياق: رموز أقل بنسبة 76% من عدم الضغط، وتعادل في أقل عدد من التكرارات",
|
||||
"en": "Context-aware compression: 76% fewer tokens than no compression, tied for fewest iterations",
|
||||
"es": "Compresión sensible al contexto: 76 % menos tokens que sin compresión y mínimo de iteraciones empatado",
|
||||
"id": "Kompresi sadar konteks: token 76% lebih sedikit dari tanpa kompresi, setara untuk iterasi paling sedikit",
|
||||
"ja": "コンテキスト対応圧縮：圧縮なしより token を76%削減、反復回数は最少タイ",
|
||||
"ko": "컨텍스트 인식 압축: 비압축보다 토큰 76% 절감, 최소 반복 횟수 공동 1위",
|
||||
"ru": "Контекстное сжатие: на 76% меньше токенов, чем без сжатия; минимум итераций разделён",
|
||||
"ta": "சூழல்-உணர்வு சுருக்கம்: சுருக்கமின்மையை விட 76% குறைந்த token; மிகக் குறைந்த சுற்றுகளில் சமநிலை",
|
||||
"tr": "Bağlam duyarlı sıkıştırma: sıkıştırmasız duruma göre %76 az token, en az iterasyonda eşit",
|
||||
"vi": "Nén theo ngữ cảnh: ít hơn 76% token so với không nén, đồng hạng ít vòng lặp nhất",
|
||||
"zhtw": "上下文感知壓縮：相比無壓縮節省 76% token，並列最少迭代次數",
|
||||
}
|
||||
|
||||
FIG17_TEXT = {
|
||||
"zh": [
|
||||
"每次搜索平均返回 ~52K 字符 → 各策略以不同方式处理",
|
||||
"166K tok · 102.1% · 失败", "277K tok · 10.9% · 12轮", "93K tok · 4.3% · 10轮",
|
||||
"40K tok · 3.0% · 7轮", "223K tok · 4.1% · 10轮", "175K tok · 102.4% · 7轮",
|
||||
],
|
||||
"ar": [
|
||||
"يُرجع كل بحث نحو 52 ألف حرف في المتوسط ← لكل استراتيجية معالجة مختلفة",
|
||||
"166K tok · 102.1% · فشل", "277K tok · 10.9% · 12 جولة", "93K tok · 4.3% · 10 جولات",
|
||||
"40K tok · 3.0% · 7 جولات", "223K tok · 4.1% · 10 جولات", "175K tok · 102.4% · 7 جولات",
|
||||
],
|
||||
"en": [
|
||||
"Each search returns ~52K characters on average → each strategy handles them differently",
|
||||
"166K tok · 102.1% · failed", "277K tok · 10.9% · 12 rounds", "93K tok · 4.3% · 10 rounds",
|
||||
"40K tok · 3.0% · 7 rounds", "223K tok · 4.1% · 10 rounds", "175K tok · 102.4% · 7 rounds",
|
||||
],
|
||||
"es": [
|
||||
"Cada búsqueda devuelve ~52 K caracteres de media → cada estrategia los procesa de forma distinta",
|
||||
"166K tok · 102,1 % · fallo", "277K tok · 10,9 % · 12 rondas", "93K tok · 4,3 % · 10 rondas",
|
||||
"40K tok · 3,0 % · 7 rondas", "223K tok · 4,1 % · 10 rondas", "175K tok · 102,4 % · 7 rondas",
|
||||
],
|
||||
"id": [
|
||||
"Setiap pencarian rata-rata mengembalikan ~52K karakter → tiap strategi menanganinya secara berbeda",
|
||||
"166K tok · 102,1% · gagal", "277K tok · 10,9% · 12 putaran", "93K tok · 4,3% · 10 putaran",
|
||||
"40K tok · 3,0% · 7 putaran", "223K tok · 4,1% · 10 putaran", "175K tok · 102,4% · 7 putaran",
|
||||
],
|
||||
"ja": [
|
||||
"各検索は平均約52K文字を返す → 戦略ごとに異なる方法で処理",
|
||||
"166K tok · 102.1% · 失敗", "277K tok · 10.9% · 12回", "93K tok · 4.3% · 10回",
|
||||
"40K tok · 3.0% · 7回", "223K tok · 4.1% · 10回", "175K tok · 102.4% · 7回",
|
||||
],
|
||||
"ko": [
|
||||
"검색당 평균 약 52K 문자를 반환 → 전략마다 다른 방식으로 처리",
|
||||
"166K tok · 102.1% · 실패", "277K tok · 10.9% · 12회", "93K tok · 4.3% · 10회",
|
||||
"40K tok · 3.0% · 7회", "223K tok · 4.1% · 10회", "175K tok · 102.4% · 7회",
|
||||
],
|
||||
"ru": [
|
||||
"Каждый поиск возвращает в среднем ~52K символов → стратегии обрабатывают их по-разному",
|
||||
"166K ток. · 102,1% · сбой", "277K ток. · 10,9% · 12 ит.", "93K ток. · 4,3% · 10 ит.",
|
||||
"40K ток. · 3,0% · 7 ит.", "223K ток. · 4,1% · 10 ит.", "175K ток. · 102,4% · 7 ит.",
|
||||
],
|
||||
"ta": [
|
||||
"ஒவ்வொரு தேடலும் சராசரியாக ~52K எழுத்துகள் → ஒவ்வொரு உத்தியும் வேறுபடச் செயலாக்கும்",
|
||||
"166K tok · 102.1% · தோல்வி", "277K tok · 10.9% · 12 சுற்று", "93K tok · 4.3% · 10 சுற்று",
|
||||
"40K tok · 3.0% · 7 சுற்று", "223K tok · 4.1% · 10 சுற்று", "175K tok · 102.4% · 7 சுற்று",
|
||||
],
|
||||
"tr": [
|
||||
"Her arama ortalama ~52K karakter döndürür → her strateji farklı biçimde işler",
|
||||
"166K tok · %102,1 · başarısız", "277K tok · %10,9 · 12 tur", "93K tok · %4,3 · 10 tur",
|
||||
"40K tok · %3,0 · 7 tur", "223K tok · %4,1 · 10 tur", "175K tok · %102,4 · 7 tur",
|
||||
],
|
||||
"vi": [
|
||||
"Mỗi lượt tìm kiếm trả về trung bình ~52K ký tự → mỗi chiến lược xử lý khác nhau",
|
||||
"166K tok · 102,1% · thất bại", "277K tok · 10,9% · 12 vòng", "93K tok · 4,3% · 10 vòng",
|
||||
"40K tok · 3,0% · 7 vòng", "223K tok · 4,1% · 10 vòng", "175K tok · 102,4% · 7 vòng",
|
||||
],
|
||||
"zhtw": [
|
||||
"每次搜尋平均返回 ~52K 字元 → 各策略以不同方式處理",
|
||||
"166K tok · 102.1% · 失敗", "277K tok · 10.9% · 12輪", "93K tok · 4.3% · 10輪",
|
||||
"40K tok · 3.0% · 7輪", "223K tok · 4.1% · 10輪", "175K tok · 102.4% · 7輪",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
TEXT_RE = re.compile(r"(<text\b[^>]*>)(.*?)(</text>)", re.DOTALL)
|
||||
|
||||
|
||||
def replace_text_nodes(svg: str, values: list[str], *, rtl: bool = False) -> str:
|
||||
matches = list(TEXT_RE.finditer(svg))
|
||||
if len(matches) != len(values):
|
||||
raise ValueError(f"expected {len(values)} text nodes, found {len(matches)}")
|
||||
replacements = iter(values)
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
opening = match.group(1)
|
||||
if rtl and "direction=" not in opening:
|
||||
opening = opening[:-1] + ' direction="rtl" unicode-bidi="plaintext">'
|
||||
return opening + html.escape(next(replacements), quote=False) + match.group(3)
|
||||
|
||||
return TEXT_RE.sub(replace, svg)
|
||||
|
||||
|
||||
def replace_text_indices(svg: str, updates: dict[int, str]) -> str:
|
||||
"""Replace selected text nodes without reserializing untouched markup."""
|
||||
index = -1
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
nonlocal index
|
||||
index += 1
|
||||
if index not in updates:
|
||||
return match.group(0)
|
||||
return match.group(1) + html.escape(updates[index], quote=False) + match.group(3)
|
||||
|
||||
output = TEXT_RE.sub(replace, svg)
|
||||
missing = set(updates) - set(range(index + 1))
|
||||
if missing:
|
||||
raise ValueError(f"missing text-node indices: {sorted(missing)}")
|
||||
return output
|
||||
|
||||
|
||||
def set_language(svg: str, locale: str) -> str:
|
||||
if "xml:lang=" in svg[:300]:
|
||||
return re.sub(r'xml:lang="[^"]+"', f'xml:lang="{locale}"', svg, count=1)
|
||||
return svg.replace("<svg ", f'<svg xml:lang="{locale}" ', 1)
|
||||
|
||||
|
||||
def set_text_attribute(locale: str, figure: int, index: int, attribute: str, value: str) -> None:
|
||||
"""Adjust one text anchor after localizing a golden layout."""
|
||||
path = ROOT / EDITIONS[locale] / "images" / f"fig2-{figure}.svg"
|
||||
svg = path.read_text(encoding="utf-8")
|
||||
current = -1
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
nonlocal current
|
||||
current += 1
|
||||
if current != index:
|
||||
return match.group(0)
|
||||
opening = re.sub(
|
||||
rf'{re.escape(attribute)}="[^"]*"',
|
||||
f'{attribute}="{value}"',
|
||||
match.group(1),
|
||||
count=1,
|
||||
)
|
||||
return opening + match.group(2) + match.group(3)
|
||||
|
||||
output = TEXT_RE.sub(replace, svg)
|
||||
if current < index:
|
||||
raise ValueError(f"missing text node {index} in {path}")
|
||||
path.write_text(output, encoding="utf-8")
|
||||
|
||||
|
||||
def sync_layout(locale: str, figure: int) -> None:
|
||||
source = (ROOT / "book" / "images" / f"fig2-{figure}.svg").read_text(encoding="utf-8")
|
||||
values = LOCALIZED_TEXT.get(locale, {}).get(figure)
|
||||
if values is None:
|
||||
values = ENGLISH_TEXT[figure]
|
||||
output = replace_text_nodes(source, values, rtl=False)
|
||||
output = set_language(output, locale)
|
||||
path = ROOT / EDITIONS[locale] / "images" / f"fig2-{figure}.svg"
|
||||
path.write_text(output.rstrip() + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def fix_figure_6(locale: str) -> None:
|
||||
path = ROOT / EDITIONS[locale] / "images" / "fig2-6.svg"
|
||||
svg = path.read_text(encoding="utf-8")
|
||||
svg = svg.replace('viewBox="0 40 760 520"', 'viewBox="0 40 760 570"')
|
||||
svg = svg.replace('width="760" height="520"', 'width="760" height="570"', 1)
|
||||
path.write_text(svg, encoding="utf-8")
|
||||
|
||||
|
||||
def fix_figure_16(locale: str) -> None:
|
||||
path = ROOT / EDITIONS[locale] / "images" / "fig2-16.svg"
|
||||
svg = path.read_text(encoding="utf-8")
|
||||
updates = {
|
||||
7: "166,043", 8: "102.1%", 9: "5",
|
||||
12: "276,608", 13: "10.9%", 14: "12",
|
||||
17: "93,449", 18: "4.3%", 19: "10",
|
||||
22: "40,157", 23: "3.0%", 24: "7",
|
||||
27: "222,992", 28: "4.1%", 29: "10",
|
||||
32: "174,601", 33: "102.4%", 34: "7",
|
||||
36: FIG16_FOOTER[locale],
|
||||
}
|
||||
if locale == "es":
|
||||
updates[3] = "Iter."
|
||||
svg = replace_text_indices(svg, updates)
|
||||
|
||||
# The 280 px visualization scale uses 280,000 tokens as its maximum, so
|
||||
# each measured 1,000 tokens corresponds to one pixel.
|
||||
widths = {
|
||||
"90": "166.043", "152": "276.608", "214": "93.449",
|
||||
"276": "40.157", "338": "222.992", "400": "174.601",
|
||||
}
|
||||
for y, width in widths.items():
|
||||
pattern = rf'(<rect x="505" y="{y}" width=")[^"]+'
|
||||
svg, count = re.subn(pattern, rf'\g<1>{width}', svg, count=1)
|
||||
if count != 1:
|
||||
raise ValueError(f"could not find Figure 2-16 bar at y={y} in {path}")
|
||||
path.write_text(svg, encoding="utf-8")
|
||||
|
||||
|
||||
def fix_figure_17(locale: str) -> None:
|
||||
path = ROOT / EDITIONS[locale] / "images" / "fig2-17.svg"
|
||||
svg = path.read_text(encoding="utf-8")
|
||||
localized = FIG17_TEXT[locale]
|
||||
updates = dict(zip((0, 4, 8, 12, 16, 20, 24), localized))
|
||||
svg = replace_text_indices(svg, updates)
|
||||
path.write_text(svg, encoding="utf-8")
|
||||
|
||||
|
||||
def normalize_arabic_text_direction() -> set[int]:
|
||||
"""Keep Arabic glyph shaping while preventing start anchors from escaping boxes."""
|
||||
changed = set()
|
||||
image_dir = ROOT / EDITIONS["ar"] / "images"
|
||||
for figure in (2, 3, 4, 5, 8, 9, 14):
|
||||
path = image_dir / f"fig2-{figure}.svg"
|
||||
svg = path.read_text(encoding="utf-8")
|
||||
fixed = svg.replace(' direction="rtl" unicode-bidi="plaintext"', "")
|
||||
if fixed != svg:
|
||||
path.write_text(fixed, encoding="utf-8")
|
||||
match = re.fullmatch(r"fig2-(\d+)\.svg", path.name)
|
||||
if match:
|
||||
changed.add(int(match.group(1)))
|
||||
return changed
|
||||
|
||||
|
||||
def synchronize(locales: list[str]) -> None:
|
||||
for locale in locales:
|
||||
changed_figures = {6, 16, 17}
|
||||
if locale in LAYOUT_SYNC_EDITIONS:
|
||||
for figure in LAYOUT_SYNC_FIGURES:
|
||||
sync_layout(locale, figure)
|
||||
changed_figures.add(figure)
|
||||
if locale == "es":
|
||||
sync_layout(locale, 10)
|
||||
sync_layout(locale, 11)
|
||||
set_text_attribute(locale, 10, 20, "x", "100")
|
||||
changed_figures.update((10, 11))
|
||||
|
||||
additional_layouts = {
|
||||
"ar": (14,),
|
||||
"id": (14,),
|
||||
"ta": (16, 17),
|
||||
"tr": (16, 17),
|
||||
"vi": (1, 10, 14, 16),
|
||||
}
|
||||
for figure in additional_layouts.get(locale, ()):
|
||||
sync_layout(locale, figure)
|
||||
changed_figures.add(figure)
|
||||
|
||||
fix_figure_6(locale)
|
||||
fix_figure_16(locale)
|
||||
fix_figure_17(locale)
|
||||
if locale == "ar":
|
||||
changed_figures.update(normalize_arabic_text_direction())
|
||||
|
||||
# Reuse the repository's idempotent overflow fitter. The Arabic copy
|
||||
# has the same geometry logic plus RTL-aware width handling; the
|
||||
# English copy is the neutral fallback for all other scripts.
|
||||
fitter_edition = "book-ar" if locale == "ar" else "book-en"
|
||||
fitter = ROOT / fitter_edition / "fit_svg_text.py"
|
||||
targets = [
|
||||
ROOT / EDITIONS[locale] / "images" / f"fig2-{figure}.svg"
|
||||
for figure in sorted(changed_figures)
|
||||
if figure != 6 # Figure 2-6 changes only its canvas height.
|
||||
]
|
||||
subprocess.run(
|
||||
[sys.executable, str(fitter), *(str(path) for path in targets)],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--locale", choices=EDITIONS, action="append", help="edition locale to update; repeatable")
|
||||
args = parser.parse_args()
|
||||
synchronize(args.locale or list(EDITIONS))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user