ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter9/hermes-self-evolution/finalize_evidence.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Finalize and independently validate the retained Experiment 9-8 evidence."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import tempfile
+
+
+ROOT = Path(__file__).resolve().parent
+RUN_ID = "exp9-8-hermes-gpt56luna-autonomous-20260802-v2"
+PINNED_COMMIT = "85c8956ec7f2b4607509980794995e1c5e21e292"
+SOURCE = ROOT / "worktree" / "hermes-agent"
+OUTPUT = ROOT / "validation" / RUN_ID
+SECRET_PATTERNS = (
+    re.compile(r"sk-or-v1-[A-Za-z0-9_-]{20,}"),
+    re.compile(r"sk-[A-Za-z0-9_-]{20,}"),
+)
+
+
+def command(args: list[str], *, cwd: Path = SOURCE) -> dict[str, object]:
+    result = subprocess.run(
+        args,
+        cwd=cwd,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        check=False,
+    )
+    return {"command": args, "exit_code": result.returncode, "output": result.stdout}
+
+
+def digest(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def safe(text: str) -> None:
+    if any(pattern.search(text) for pattern in SECRET_PATTERNS):
+        raise RuntimeError("credential-shaped value detected in retained evidence")
+
+
+def build_patch() -> str:
+    patch = str(command(["git", "diff", "--binary", "--", "."])["output"])
+    names = str(command(["git", "ls-files", "--others", "--exclude-standard"])["output"])
+    for name in names.splitlines():
+        path = SOURCE / name
+        if not path.is_file():
+            continue
+        addition = command(
+            ["git", "diff", "--no-index", "--binary", "--", "/dev/null", name]
+        )
+        patch += str(addition["output"])
+    safe(patch)
+    return patch
+
+
+def main() -> int:
+    if str(command(["git", "rev-parse", "HEAD"])["output"]).strip() != PINNED_COMMIT:
+        raise RuntimeError("Hermes worktree is not at the pinned baseline")
+
+    checks = [
+        command(["uv", "run", "--with", "pytest", "pytest", "tests/agent/test_trajectory.py", "-q"]),
+        command(
+            [
+                "uv", "run", "--with", "pytest", "pytest",
+                "tests/test_batch_runner_checkpoint.py",
+                "tests/test_batch_runner_durability.py",
+                "tests/integration/test_batch_runner.py",
+                "tests/test_trajectory_compressor.py", "-q",
+            ]
+        ),
+        command(
+            [
+                "python3", "-m", "py_compile", "agent/trajectory.py",
+                "agent/agent_runtime_helpers.py", "batch_runner.py", "run_agent.py",
+                "mini_swe_runner.py", "tests/agent/test_trajectory.py",
+            ]
+        ),
+        command(["git", "diff", "--check"]),
+    ]
+    if any(int(check["exit_code"]) != 0 for check in checks):
+        raise RuntimeError("independent validation failed")
+
+    patch_path = OUTPUT / "hermes-self-evolution.patch"
+    patch_path.write_text(build_patch(), encoding="utf-8")
+    report_path = OUTPUT / "BOOK_SELF_EVOLUTION_REPORT.md"
+    shutil.copyfile(SOURCE / "BOOK_SELF_EVOLUTION_REPORT.md", report_path)
+
+    with tempfile.TemporaryDirectory(prefix="hermes-patch-check-") as temp:
+        clean = Path(temp) / "hermes-agent"
+        cloned = command(["git", "clone", "--shared", str(SOURCE), str(clean)], cwd=ROOT)
+        if int(cloned["exit_code"]) != 0:
+            raise RuntimeError(str(cloned["output"]))
+        applied = command(["git", "apply", "--check", str(patch_path)], cwd=clean)
+        if int(applied["exit_code"]) != 0:
+            raise RuntimeError(f"retained patch is invalid: {applied['output']}")
+
+    transcript_names = ["hermes-transcript.txt"]
+    transcript_names.extend(
+        f"hermes-review-autonomous-{round_number}.txt"
+        for round_number in range(1, 4)
+    )
+    transcript_names.extend(
+        f"hermes-acceptance-review-{round_number}.txt"
+        for round_number in range(1, 5)
+    )
+    transcript_hashes = {}
+    for name in transcript_names:
+        path = OUTPUT / "raw" / name
+        text = path.read_text(encoding="utf-8")
+        safe(text)
+        transcript_hashes[name] = digest(path)
+    safe(report_path.read_text(encoding="utf-8"))
+    terminal_review = (OUTPUT / "raw" / "hermes-acceptance-review-4.txt").read_text(
+        encoding="utf-8"
+    )
+    verdicts = re.findall(r"^VERDICT: (ACCEPT|REJECT)$", terminal_review, re.MULTILINE)
+    if not verdicts or verdicts[-1] != "ACCEPT":
+        raise RuntimeError("terminal acceptance review did not accept the candidate")
+
+    manifest = {
+        "schema_version": 2,
+        "experiment": "9-8",
+        "run_id": RUN_ID,
+        "source_repository": "https://github.com/NousResearch/hermes-agent.git",
+        "started_from_commit": PINNED_COMMIT,
+        "provider": "openrouter",
+        "requested_model": "openai/gpt-5.6-luna",
+        "credential_environment_variable": "OPENROUTER_API_KEY",
+        "candidate_gaps_supplied_in_prompt": False,
+        "task_prompt_sha256": digest(ROOT / "task.md"),
+        "proposer_exit_codes": [0] * 4,
+        "acceptance_reviewer_exit_codes": [3, 3, 3, 0],
+        "interaction_rounds": 4,
+        "independent_acceptance_reviews": 4,
+        "terminal_reviewer_verdict": "ACCEPT",
+        "review_findings_corrected": [
+            "the first parser did not understand production XML-wrapped tool responses",
+            "batch and sample trajectory writers initially omitted the evaluation metadata",
+            "one failed result could be double-counted",
+            "the Mini-SWE trajectory writer initially remained outside the shared contract",
+        ],
+        "final_candidate": {
+            "autonomously_selected": "evidence-backed learning signals for persisted trajectories",
+            "implemented": (
+                "conservative evaluation metadata shared across standard, batch, sample, "
+                "and Mini-SWE trajectory persistence paths"
+            ),
+            "deferred": [
+                "automatic mutation from a single trajectory",
+                "product-level ablation campaign runner",
+                "generic multi-agent reviewer without an artifact contract",
+            ],
+            "status": "candidate_patch_accepted_by_terminal_reviewer_not_merged",
+        },
+        "independent_checks": checks,
+        "patch_apply_check": "passed",
+        "patch_sha256": digest(patch_path),
+        "report_sha256": digest(report_path),
+        "transcript_sha256": transcript_hashes,
+        "credential_scan": "passed",
+        "claim_boundary": (
+            "The run demonstrates autonomous audit, candidate generation, repeated correction under "
+            "independent rejection, and terminal acceptance. It does not demonstrate downstream "
+            "task-quality uplift; the proposed ablation campaign was not run."
+        ),
+    }
+    manifest_text = json.dumps(manifest, ensure_ascii=False, indent=2) + "\n"
+    safe(manifest_text)
+    (OUTPUT / "manifest.json").write_text(manifest_text, encoding="utf-8")
+    print(manifest_text)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())