ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter7/openvla-robotwin2-eval/experiment.py

@@ -0,0 +1,783 @@
+#!/usr/bin/env python3
+"""Exact companion for manuscript Experiment 7-13.
+
+The runner deliberately separates three things:
+1. a non-destructive host/upstream preflight;
+2. generation or execution of the two real upstream val-only arms; and
+3. strict analysis of episode-level evidence.
+
+Paper numbers, historical rollout collections and dry-run commands never satisfy
+the completion gate.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import re
+import shutil
+import statistics
+import subprocess
+import sys
+from collections import Counter
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterable
+
+
+HERE = Path(__file__).resolve().parent
+DEFAULT_CONFIG = HERE / "config.json"
+VIDEO_RE = re.compile(
+    r"step=(?P<global_step>\d+)--task=(?P<task>.+?)--success=(?P<success>True|False)--ran=(?P<run>[^.]+)\.mp4$"
+)
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def load_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def dump_json(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+def resolve_config(path: Path) -> dict[str, Any]:
+    cfg = load_json(path)
+    upstream = Path(cfg["upstream_path"])
+    if not upstream.is_absolute():
+        upstream = (path.parent / upstream).resolve()
+    cfg["upstream_path"] = str(upstream)
+    return cfg
+
+
+def run_capture(argv: list[str], cwd: Path | None = None) -> tuple[int, str]:
+    try:
+        proc = subprocess.run(
+            argv,
+            cwd=cwd,
+            check=False,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
+        return proc.returncode, proc.stdout.strip()
+    except FileNotFoundError as exc:
+        return 127, f"{type(exc).__name__}: {exc}"
+
+
+def gpu_inventory() -> tuple[list[dict[str, Any]], str | None]:
+    code, output = run_capture(
+        [
+            "nvidia-smi",
+            "--query-gpu=index,name,memory.total,driver_version",
+            "--format=csv,noheader,nounits",
+        ]
+    )
+    if code != 0:
+        return [], output or "nvidia-smi unavailable"
+    gpus = []
+    for line in output.splitlines():
+        parts = [part.strip() for part in line.split(",")]
+        if len(parts) >= 4:
+            gpus.append(
+                {
+                    "index": int(parts[0]),
+                    "name": parts[1],
+                    "memory_mib": int(parts[2]),
+                    "driver": parts[3],
+                }
+            )
+    return gpus, None
+
+
+def preflight(cfg: dict[str, Any]) -> dict[str, Any]:
+    upstream = Path(cfg["upstream_path"])
+    checks: dict[str, dict[str, Any]] = {}
+
+    def check(name: str, ok: bool, evidence: Any, required: bool = True) -> None:
+        checks[name] = {"ok": bool(ok), "required": required, "evidence": evidence}
+
+    check("upstream_checkout", (upstream / ".git").exists(), str(upstream))
+    commit_code, commit = run_capture(["git", "rev-parse", "HEAD"], cwd=upstream)
+    check(
+        "pinned_upstream_commit",
+        commit_code == 0 and commit == cfg["expected_upstream_commit"],
+        {"observed": commit, "expected": cfg["expected_upstream_commit"]},
+    )
+
+    required_files = [
+        "examples/run_openvla_oft_rl_twin2.sh",
+        "examples/robotwin2_tasks_info.txt",
+        "verl/trainer/main_ppo.py",
+        "verl/trainer/ppo/ray_trainer.py",
+        "verl/utils/dataset/rob_dataset.py",
+        "verl/workers/rollout/rob_rollout.py",
+        "modified_codes/robotwin2/envs/move_can_pot.py",
+        "modified_codes/robotwin2/task_config/demo_randomized.yml",
+        "verl/utils/envs/robotwin2/seeds/robotwin2_eval_seeds.json",
+    ]
+    missing = [item for item in required_files if not (upstream / item).is_file()]
+    check("required_upstream_files", not missing, {"missing": missing, "required": required_files})
+
+    task_info = upstream / "examples/robotwin2_tasks_info.txt"
+    task_supported = task_info.is_file() and cfg["task"] in task_info.read_text(encoding="utf-8")
+    check("move_can_pot_supported", task_supported, str(task_info))
+
+    seed_file = upstream / "verl/utils/envs/robotwin2/seeds/robotwin2_eval_seeds.json"
+    observed_seed_count = 0
+    if seed_file.is_file():
+        seed_data = load_json(seed_file)
+        observed_seed_count = len(seed_data.get(cfg["task"], {}).get("success_seeds", []))
+    check(
+        "ood_seed_inventory",
+        observed_seed_count >= cfg["ood_validation_seeds"],
+        {"observed": observed_seed_count, "required": cfg["ood_validation_seeds"]},
+    )
+
+    rollout_source = upstream / "verl/workers/rollout/rob_rollout.py"
+    rollout_text = rollout_source.read_text(encoding="utf-8") if rollout_source.is_file() else ""
+    source_invariants = {
+        "three_view_branch": "num_images_in_input == 3" in rollout_text,
+        "left_wrist": '"left_wrist"' in rollout_text,
+        "right_wrist": '"right_wrist"' in rollout_text,
+        "proprio_vector": "joint_action']['vector" in rollout_text,
+        "environment_success": "self.env.eval_success" in rollout_text,
+        "validation_video": "save_rollout_video" in rollout_text,
+    }
+    check("source_invariants", all(source_invariants.values()), source_invariants)
+
+    checkpoint_raw = os.environ.get(cfg["checkpoint_env"], "")
+    checkpoint = Path(checkpoint_raw).expanduser() if checkpoint_raw else None
+    checkpoint_metadata: dict[str, Any] = {
+        "environment": cfg["checkpoint_env"],
+        "path": checkpoint_raw or None,
+        "checkpoint_id": cfg["checkpoint_id"],
+        "expected_revision": cfg["expected_checkpoint_revision"],
+    }
+    checkpoint_ready = bool(checkpoint and checkpoint.is_dir())
+    if checkpoint_ready and checkpoint is not None:
+        metadata_dir = checkpoint / ".cache/huggingface/download"
+        primary_names = [
+            "config.json",
+            "model-00001-of-00004.safetensors",
+            "model-00002-of-00004.safetensors",
+            "model-00003-of-00004.safetensors",
+            "model-00004-of-00004.safetensors",
+            "proprio_projector--20000_checkpoint.pt",
+            "lora_adapter/adapter_model.safetensors",
+        ]
+        artifact_metadata = {}
+        observed_revisions = set()
+        missing_artifacts = []
+        for name in primary_names:
+            artifact = checkpoint / name
+            metadata = metadata_dir / f"{name}.metadata"
+            if not artifact.is_file() or not metadata.is_file():
+                missing_artifacts.append(name)
+                continue
+            lines = metadata.read_text(encoding="utf-8").splitlines()
+            if len(lines) < 2:
+                missing_artifacts.append(name)
+                continue
+            observed_revisions.add(lines[0])
+            artifact_metadata[name] = {
+                "bytes": artifact.stat().st_size,
+                "huggingface_etag": lines[1],
+            }
+        checkpoint_metadata.update({
+            "observed_revisions": sorted(observed_revisions),
+            "artifact_metadata": artifact_metadata,
+            "missing_artifacts": missing_artifacts,
+        })
+        checkpoint_ready = (
+            not missing_artifacts
+            and observed_revisions == {cfg["expected_checkpoint_revision"]}
+        )
+    check(
+        "pretrained_checkpoint",
+        checkpoint_ready,
+        checkpoint_metadata,
+    )
+
+    robotwin_raw = os.environ.get(cfg["robotwin2_env"], "")
+    robotwin = Path(robotwin_raw).expanduser() if robotwin_raw else None
+    robotwin_markers = ["envs", "task_config", "script"]
+    check(
+        "robotwin2_checkout",
+        bool(robotwin and robotwin.is_dir() and all((robotwin / marker).exists() for marker in robotwin_markers)),
+        {"environment": cfg["robotwin2_env"], "path": robotwin_raw or None, "markers": robotwin_markers},
+    )
+    robotwin_code, robotwin_commit = (
+        run_capture(["git", "rev-parse", "HEAD"], cwd=robotwin)
+        if robotwin and robotwin.is_dir()
+        else (1, "")
+    )
+    check(
+        "pinned_robotwin2_commit",
+        robotwin_code == 0 and robotwin_commit == cfg["expected_robotwin2_commit"],
+        {"observed": robotwin_commit or None, "expected": cfg["expected_robotwin2_commit"]},
+    )
+
+    align_raw = os.environ.get(cfg["align_path_env"], "")
+    align = Path(align_raw).expanduser() if align_raw else upstream / "align.json"
+    check(
+        "ray_runtime_environment",
+        align.is_file(),
+        {"environment": cfg["align_path_env"], "path": str(align)},
+    )
+
+    gpus, gpu_error = gpu_inventory()
+    check(
+        "nvidia_gpu_count",
+        len(gpus) >= cfg["gpus_required"],
+        {"observed": len(gpus), "required": cfg["gpus_required"], "gpus": gpus, "error": gpu_error},
+    )
+
+    custom_task_config = HERE / "task_config_exp7_13_three_view.yml"
+    task_config_text = custom_task_config.read_text(encoding="utf-8") if custom_task_config.is_file() else ""
+    check(
+        "three_rgb_view_config",
+        custom_task_config.is_file()
+        and "collect_head_camera: true" in task_config_text
+        and "collect_wrist_camera: true" in task_config_text,
+        str(custom_task_config),
+    )
+
+    required_failures = [name for name, item in checks.items() if item["required"] and not item["ok"]]
+    return {
+        "schema_version": 1,
+        "experiment": cfg["experiment"],
+        "generated_at_utc": utc_now(),
+        "host": {
+            "platform": sys.platform,
+            "python": sys.version.split()[0],
+            "cwd": str(Path.cwd()),
+        },
+        "configuration": cfg,
+        "checks": checks,
+        "ready_for_real_validation": not required_failures,
+        "blocking_checks": required_failures,
+        "acceptance_note": "Preflight readiness is necessary but never sufficient for Experiment 7-13 completion.",
+    }
+
+
+def hydra_command(cfg: dict[str, Any], arm: str, run_dir: Path, worktree: Path) -> list[str]:
+    chunk = int(arm.removeprefix("chunk_"))
+    checkpoint = str(Path(os.environ[cfg["checkpoint_env"]]).expanduser().resolve())
+    align_raw = os.environ.get(cfg["align_path_env"])
+    align_path = Path(align_raw).expanduser().resolve() if align_raw else worktree / "align.json"
+    experiment_name = f"exp7_13_{cfg['task']}_{arm}"
+    return [
+        sys.executable,
+        "-u",
+        "-m",
+        "verl.trainer.main_ppo",
+        f"data.task_suite_name={cfg['task_suite']}",
+        "data.num_trials_per_task=128",
+        "data.n_samples=1",
+        "data.filter_accuracy=False",
+        "data.train_batch_size=64",
+        "data.val_batch_size=8",
+        "data.max_prompt_length=256",
+        "data.max_response_length=128",
+        f"actor_rollout_ref.model.path={checkpoint}",
+        f"actor_rollout_ref.model.vla={cfg['vla']}",
+        f"actor_rollout_ref.model.action_token_len={cfg['action_token_length']}",
+        f"actor_rollout_ref.model.action_chunks_len={chunk}",
+        "actor_rollout_ref.model.resume=False",
+        f"actor_rollout_ref.actor.num_images_in_input={cfg['rgb_views']}",
+        "actor_rollout_ref.actor.traj_mini_batch_size=8",
+        "actor_rollout_ref.actor.ppo_mini_batch_size=128",
+        "actor_rollout_ref.actor.ppo_micro_batch_size=8",
+        "actor_rollout_ref.actor.use_dynamic_bsz=False",
+        "actor_rollout_ref.actor.fsdp_config.param_offload=False",
+        "actor_rollout_ref.actor.fsdp_config.grad_offload=True",
+        "actor_rollout_ref.actor.fsdp_config.optimizer_offload=True",
+        f"actor_rollout_ref.rollout.twin2_task_config={cfg['task_config']}",
+        "actor_rollout_ref.rollout.twin2_instruction_type=seen",
+        f"actor_rollout_ref.rollout.num_images_in_input={cfg['rgb_views']}",
+        f"+actor_rollout_ref.rollout.action_token_len={cfg['action_token_length']}",
+        "actor_rollout_ref.rollout.use_proprio=True",
+        "actor_rollout_ref.rollout.val_micro_batch_size=8",
+        "actor_rollout_ref.rollout.temperature=1.6",
+        f"actor_rollout_ref.rollout.experiment_name={experiment_name}",
+        "actor_rollout_ref.rollout.micro_batch_size=1",
+        f"actor_rollout_ref.rollout.unnorm_key=robotwin2_{cfg['task']}_1k",
+        f"actor_rollout_ref.rollout.model_family={cfg['model_family']}",
+        f"actor_rollout_ref.rollout.task_suite_name={cfg['task_suite']}",
+        "actor_rollout_ref.rollout.num_steps_wait=10",
+        f"actor_rollout_ref.rollout.pretrained_checkpoint={checkpoint}",
+        "actor_rollout_ref.rollout.center_crop=True",
+        "actor_rollout_ref.rollout.max_prompt_length=512",
+        "actor_rollout_ref.rollout.tensor_model_parallel_size=1",
+        "actor_rollout_ref.rollout.name=hf",
+        "actor_rollout_ref.rollout.gpu_memory_utilization=0.9",
+        "actor_rollout_ref.ref.log_prob_micro_batch_size=32",
+        "actor_rollout_ref.ref.fsdp_config.param_offload=True",
+        "algorithm.adv_estimator=grpo",
+        "algorithm.kl_ctrl.kl_coef=0.0",
+        "trainer.logger=['console']",
+        "trainer.project_name=AI-Agent-Book-Experiment-7-13",
+        f"trainer.experiment_name={experiment_name}",
+        f"trainer.default_local_dir={str((run_dir / arm / 'checkpoints').resolve())}",
+        f"trainer.n_gpus_per_node={cfg['gpus_required']}",
+        "trainer.nnodes=1",
+        "trainer.save_freq=-1",
+        "trainer.test_freq=-1",
+        "trainer.total_epochs=1",
+        "trainer.val_only=True",
+        "trainer.val_before_train=True",
+        f"trainer.runtime_env={str(align_path)}",
+        "trainer.wandb_mode=disabled",
+    ]
+
+
+def shell_quote(value: str) -> str:
+    import shlex
+
+    return shlex.quote(value)
+
+
+def prepare_worktree(cfg: dict[str, Any], run_dir: Path) -> Path:
+    upstream = Path(cfg["upstream_path"])
+    worktree = run_dir / "instrumented-upstream"
+    if worktree.exists():
+        raise RuntimeError(f"refusing to replace existing worktree: {worktree}")
+    code, output = run_capture(
+        ["git", "worktree", "add", "--detach", str(worktree), cfg["expected_upstream_commit"]],
+        cwd=upstream,
+    )
+    if code != 0:
+        raise RuntimeError(f"git worktree add failed: {output}")
+    robotwin = Path(os.environ[cfg["robotwin2_env"]]).expanduser().resolve()
+    setup = subprocess.run(
+        ["bash", "copy_overwrite_robotwin2.sh", str(robotwin), str(worktree)],
+        cwd=worktree,
+        check=False,
+    )
+    if setup.returncode != 0:
+        raise RuntimeError(f"RoboTwin2 overlay setup failed with exit {setup.returncode}")
+    task_config_dest = worktree / "verl/utils/envs/robotwin2/task_config" / f"{cfg['task_config']}.yml"
+    task_config_dest.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(HERE / "task_config_exp7_13_three_view.yml", task_config_dest)
+    patch_proc = subprocess.run(
+        [sys.executable, str(HERE / "instrument_upstream.py"), str(worktree)],
+        check=False,
+    )
+    if patch_proc.returncode != 0:
+        raise RuntimeError(f"upstream instrumentation failed with exit {patch_proc.returncode}")
+    return worktree
+
+
+def write_launch_manifest(cfg: dict[str, Any], run_dir: Path, worktree: Path) -> dict[str, Any]:
+    arms = {}
+    for chunk in cfg["action_chunks"]:
+        arm = f"chunk_{chunk}"
+        argv = hydra_command(cfg, arm, run_dir, worktree)
+        episode_path = (run_dir / arm / "episodes.jsonl").resolve()
+        env = {
+            "EXP7_13_EPISODE_JSONL": str(episode_path),
+            "EXP7_13_ARM": arm,
+            "EXP7_13_UPSTREAM_COMMIT": cfg["expected_upstream_commit"],
+            "HYDRA_FULL_ERROR": "1",
+            "TOKENIZERS_PARALLELISM": "true",
+            "NCCL_DEBUG": "WARN",
+            "ROBOT_PLATFORM": "ALOHA",
+            "VERL_DISABLE_VLLM_IMPORT": "1",
+        }
+        arms[arm] = {
+            "action_chunk_length": chunk,
+            "episode_evidence": str(episode_path),
+            "rollout_directory": str((worktree / "rollouts" / f"exp7_13_{cfg['task']}_{arm}").resolve()),
+            "environment": env,
+            "argv": argv,
+            "shell_command": " ".join(
+                [*(f"{key}={shell_quote(value)}" for key, value in env.items()), *(shell_quote(item) for item in argv)]
+            ),
+        }
+    manifest = {
+        "schema_version": 1,
+        "experiment": cfg["experiment"],
+        "generated_at_utc": utc_now(),
+        "real_execution_required": True,
+        "upstream_worktree": str(worktree.resolve()),
+        "upstream_commit": cfg["expected_upstream_commit"],
+        "arms": arms,
+    }
+    dump_json(run_dir / "launch_manifest.json", manifest)
+    return manifest
+
+
+def execute_arms(manifest: dict[str, Any], selected: Iterable[str]) -> None:
+    worktree = Path(manifest["upstream_worktree"])
+    for arm in selected:
+        spec = manifest["arms"][arm]
+        arm_dir = Path(spec["episode_evidence"]).parent
+        arm_dir.mkdir(parents=True, exist_ok=True)
+        log_path = arm_dir / "upstream.log"
+        env = os.environ.copy()
+        env.update(spec["environment"])
+        started = utc_now()
+        with log_path.open("w", encoding="utf-8") as log:
+            proc = subprocess.run(
+                spec["argv"],
+                cwd=worktree,
+                env=env,
+                check=False,
+                stdout=log,
+                stderr=subprocess.STDOUT,
+            )
+        dump_json(
+            arm_dir / "process.json",
+            {
+                "arm": arm,
+                "started_at_utc": started,
+                "ended_at_utc": utc_now(),
+                "exit_code": proc.returncode,
+                "log": str(log_path.resolve()),
+            },
+        )
+        if proc.returncode != 0:
+            raise RuntimeError(f"{arm} real validation failed with exit {proc.returncode}; see {log_path}")
+
+
+def percentile(values: list[float], q: float) -> float | None:
+    if not values:
+        return None
+    ordered = sorted(values)
+    if len(ordered) == 1:
+        return ordered[0]
+    pos = (len(ordered) - 1) * q
+    lo = math.floor(pos)
+    hi = math.ceil(pos)
+    if lo == hi:
+        return ordered[lo]
+    return ordered[lo] + (ordered[hi] - ordered[lo]) * (pos - lo)
+
+
+def distribution(values: list[float]) -> dict[str, float | int | None]:
+    return {
+        "count": len(values),
+        "mean": statistics.fmean(values) if values else None,
+        "std": statistics.pstdev(values) if len(values) > 1 else 0.0 if values else None,
+        "min": min(values) if values else None,
+        "p50": percentile(values, 0.50),
+        "p95": percentile(values, 0.95),
+        "max": max(values) if values else None,
+    }
+
+
+def read_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows = []
+    if not path.is_file():
+        return rows
+    for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        if not line.strip():
+            continue
+        try:
+            rows.append(json.loads(line))
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"{path}:{line_no}: {exc}") from exc
+    return rows
+
+
+def video_index(
+    directory: Path,
+    *,
+    started_at_utc: str | None = None,
+    ended_at_utc: str | None = None,
+) -> dict[tuple[str, bool], list[str]]:
+    """Index rollout videos, optionally restricting them to one process window.
+
+    Upstream names videos with a random ``ran`` suffix and does not clear an
+    existing rollout directory.  A bounded resume can therefore leave two
+    files for the same task/result key.  Process timestamps let the evidence
+    reader reject those stale files without guessing from the random suffix.
+    """
+    index: dict[tuple[str, bool], list[str]] = {}
+    if not directory.is_dir():
+        return index
+    started = datetime.fromisoformat(started_at_utc) if started_at_utc else None
+    ended = datetime.fromisoformat(ended_at_utc) if ended_at_utc else None
+    for path in directory.rglob("*.mp4"):
+        modified = datetime.fromtimestamp(path.stat().st_mtime, timezone.utc)
+        if started is not None and modified < started:
+            continue
+        if ended is not None and modified > ended:
+            continue
+        match = VIDEO_RE.match(path.name)
+        if not match:
+            continue
+        task_key = match.group("task")
+        success = match.group("success") == "True"
+        index.setdefault((task_key, success), []).append(str(path.resolve()))
+    return index
+
+
+def annotation_key(row: dict[str, Any]) -> str:
+    return f"{row['arm']}|{row['data_source']}|{row['trial_seed']}"
+
+
+def analyze(cfg: dict[str, Any], run_dir: Path, annotations_path: Path | None) -> dict[str, Any]:
+    manifest_path = run_dir / "launch_manifest.json"
+    manifest = load_json(manifest_path) if manifest_path.is_file() else None
+    annotations = load_json(annotations_path) if annotations_path and annotations_path.is_file() else {}
+    all_rows: list[dict[str, Any]] = []
+    arm_reports: dict[str, Any] = {}
+    strict_errors: list[str] = []
+
+    expected_total = cfg["iid_validation_seeds"] + cfg["ood_validation_seeds"]
+    for chunk in cfg["action_chunks"]:
+        arm = f"chunk_{chunk}"
+        episode_path = run_dir / arm / "episodes.jsonl"
+        rows = read_jsonl(episode_path)
+        all_rows.extend(rows)
+        bad_source = [row for row in rows if row.get("source") != "upstream_val_only"]
+        if bad_source:
+            strict_errors.append(f"{arm}: {len(bad_source)} rows are not upstream_val_only evidence")
+        if len(rows) != expected_total:
+            strict_errors.append(f"{arm}: expected {expected_total} episodes, observed {len(rows)}")
+        if any(row.get("action_chunk_length") != chunk for row in rows):
+            strict_errors.append(f"{arm}: action chunk mismatch")
+        if any(row.get("action_dimension") != cfg["action_dimension"] for row in rows):
+            strict_errors.append(f"{arm}: action dimension mismatch")
+        if any(row.get("rgb_views") != cfg["rgb_views"] for row in rows):
+            strict_errors.append(f"{arm}: three-view observation not proven")
+        if any(not row.get("proprioception_enabled") for row in rows):
+            strict_errors.append(f"{arm}: proprioception not enabled in all rows")
+
+        iid = [row for row in rows if str(row.get("data_source", "")).endswith("_train_iid")]
+        ood = [row for row in rows if str(row.get("data_source", "")).endswith("_eval_ood")]
+        if len({row.get("trial_seed") for row in iid}) != cfg["iid_validation_seeds"]:
+            strict_errors.append(f"{arm}: incomplete IID seed coverage")
+        if len({row.get("trial_seed") for row in ood}) != cfg["ood_validation_seeds"]:
+            strict_errors.append(f"{arm}: incomplete OOD seed coverage")
+
+        video_dir = Path(manifest["arms"][arm]["rollout_directory"]) if manifest else Path()
+        process_path = run_dir / arm / "process.json"
+        process_data = load_json(process_path) if process_path.is_file() else {}
+        videos = video_index(
+            video_dir,
+            started_at_utc=process_data.get("started_at_utc"),
+            ended_at_utc=process_data.get("ended_at_utc"),
+        )
+        unmatched_videos = 0
+        failure_counts: Counter[str] = Counter()
+        unclassified_failures = []
+        missing_video_count = 0
+        for row in rows:
+            task_file = f"{cfg['task']}_trial_{row.get('trial_id')}_seed_{row.get('trial_seed')}"
+            candidates = videos.get((task_file, bool(row.get("success"))), [])
+            if candidates:
+                row["video_path"] = candidates.pop(0)
+            else:
+                missing_video_count += 1
+            if row.get("success"):
+                row["failure_mode"] = None
+                row["failure_evidence"] = None
+            else:
+                annotation = annotations.get(annotation_key(row))
+                if annotation:
+                    label = annotation.get("failure_mode")
+                    evidence = annotation.get("evidence")
+                    if label not in cfg["failure_modes"] or not evidence:
+                        unclassified_failures.append(annotation_key(row))
+                    else:
+                        row["failure_mode"] = label
+                        row["failure_evidence"] = evidence
+                        failure_counts[label] += 1
+                else:
+                    unclassified_failures.append(annotation_key(row))
+        unmatched_videos = sum(len(paths) for paths in videos.values())
+        if missing_video_count:
+            strict_errors.append(f"{arm}: {missing_video_count} episodes lack rollout video evidence")
+        if unclassified_failures:
+            strict_errors.append(f"{arm}: {len(unclassified_failures)} failed episodes lack valid failure annotations")
+
+        action_steps = [float(row["finish_action_steps"]) for row in rows if row.get("finish_action_steps") is not None]
+        control_seconds = [value / cfg["control_hz"] for value in action_steps]
+        successes = sum(bool(row.get("success")) for row in rows)
+        by_split = {}
+        for split_name, split_rows in (("iid", iid), ("ood", ood)):
+            split_successes = sum(bool(row.get("success")) for row in split_rows)
+            by_split[split_name] = {
+                "episodes": len(split_rows),
+                "successes": split_successes,
+                "success_rate": split_successes / len(split_rows) if split_rows else None,
+            }
+        arm_reports[arm] = {
+            "action_chunk_length": chunk,
+            "episodes": len(rows),
+            "successes": successes,
+            "success_rate": successes / len(rows) if rows else None,
+            "by_split": by_split,
+            "completion_action_steps": distribution(action_steps),
+            "completion_control_seconds_at_50hz": distribution(control_seconds),
+            "failure_modes": dict(sorted(failure_counts.items())),
+            "unclassified_failure_count": len(unclassified_failures),
+            "missing_video_count": missing_video_count,
+            "unmatched_video_count": unmatched_videos,
+        }
+
+    control = arm_reports.get("chunk_1", {})
+    treatment = arm_reports.get("chunk_25", {})
+    paired = {}
+    rows_by_key = {
+        (row.get("arm"), row.get("data_source"), row.get("trial_seed")): row for row in all_rows
+    }
+    pair_deltas = []
+    for source in ("train_iid", "eval_ood"):
+        suffix = f"_{source}"
+        seeds = {
+            row.get("trial_seed")
+            for row in all_rows
+            if str(row.get("data_source", "")).endswith(suffix)
+        }
+        for seed in seeds:
+            left = next(
+                (
+                    row
+                    for row in all_rows
+                    if row.get("arm") == "chunk_1"
+                    and str(row.get("data_source", "")).endswith(suffix)
+                    and row.get("trial_seed") == seed
+                ),
+                None,
+            )
+            right = next(
+                (
+                    row
+                    for row in all_rows
+                    if row.get("arm") == "chunk_25"
+                    and str(row.get("data_source", "")).endswith(suffix)
+                    and row.get("trial_seed") == seed
+                ),
+                None,
+            )
+            if left and right:
+                pair_deltas.append(int(bool(right.get("success"))) - int(bool(left.get("success"))))
+    paired = {
+        "complete_pairs": len(pair_deltas),
+        "chunk_25_wins": sum(delta > 0 for delta in pair_deltas),
+        "chunk_1_wins": sum(delta < 0 for delta in pair_deltas),
+        "ties": sum(delta == 0 for delta in pair_deltas),
+        "mean_paired_success_delta": statistics.fmean(pair_deltas) if pair_deltas else None,
+        "unpaired_rows": len(all_rows) - 2 * len(pair_deltas),
+    }
+    if pair_deltas and len(pair_deltas) != expected_total:
+        strict_errors.append(f"paired comparison: expected {expected_total} pairs, observed {len(pair_deltas)}")
+
+    preflight_files = sorted((run_dir / "preflight.json", *HERE.glob("results/preflight-*.json")), key=lambda p: p.stat().st_mtime if p.exists() else 0)
+    preflight_report = load_json(preflight_files[-1]) if preflight_files and preflight_files[-1].is_file() else None
+    if not preflight_report or not preflight_report.get("ready_for_real_validation"):
+        strict_errors.append("no passing real-execution preflight is attached to this run")
+    process_failures = []
+    for chunk in cfg["action_chunks"]:
+        arm = f"chunk_{chunk}"
+        process_path = run_dir / arm / "process.json"
+        process_data = load_json(process_path) if process_path.is_file() else None
+        if not process_data or process_data.get("exit_code") != 0:
+            process_failures.append(arm)
+    if process_failures:
+        strict_errors.append(f"missing or failed upstream processes: {', '.join(process_failures)}")
+
+    report = {
+        "schema_version": 1,
+        "experiment": cfg["experiment"],
+        "generated_at_utc": utc_now(),
+        "evidence_policy": {
+            "paper_numbers_accepted": False,
+            "historical_rollouts_accepted": False,
+            "dry_run_commands_accepted": False,
+            "required_source": "instrumented upstream val-only execution",
+        },
+        "configuration": cfg,
+        "arms": arm_reports,
+        "controlled_action_chunk_comparison": paired,
+        "strict_completion": {
+            "complete": not strict_errors,
+            "errors": strict_errors,
+        },
+    }
+    dump_json(run_dir / "report.json", report)
+    lines = [
+        "# Experiment 7-13 OpenVLA + RoboTwin2 evaluation",
+        "",
+        f"Official completion: **{report['strict_completion']['complete']}**",
+        "",
+        "| Arm | Episodes | Success rate | IID | OOD | Completion p50 / p95 (50 Hz seconds) |",
+        "| --- | ---: | ---: | ---: | ---: | ---: |",
+    ]
+    for arm, item in arm_reports.items():
+        times = item["completion_control_seconds_at_50hz"]
+        lines.append(
+            f"| {arm} | {item['episodes']} | {item['success_rate']} | "
+            f"{item['by_split']['iid']['success_rate']} | {item['by_split']['ood']['success_rate']} | "
+            f"{times['p50']} / {times['p95']} |"
+        )
+    lines.extend(["", "## Controlled comparison", "", f"```json\n{json.dumps(paired, indent=2)}\n```", "", "## Failure modes", ""])
+    for arm, item in arm_reports.items():
+        lines.append(f"- {arm}: {json.dumps(item['failure_modes'], ensure_ascii=False)}")
+    lines.extend(["", "## Completion audit", ""])
+    if strict_errors:
+        lines.extend(f"- {error}" for error in strict_errors)
+    else:
+        lines.append("- Every direct manuscript gate is supported by real per-episode evidence.")
+    (run_dir / "report.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return report
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG)
+    sub = parser.add_subparsers(dest="command", required=True)
+    pre = sub.add_parser("preflight", help="inspect the real host and upstream checkout")
+    pre.add_argument("--output", type=Path)
+    prep = sub.add_parser("prepare", help="prepare an instrumented disposable worktree and commands")
+    prep.add_argument("--run-dir", type=Path, required=True)
+    launch = sub.add_parser("launch", help="execute real val-only arms from an existing manifest")
+    launch.add_argument("--run-dir", type=Path, required=True)
+    launch.add_argument("--arm", choices=["all", "chunk_1", "chunk_25"], default="all")
+    ana = sub.add_parser("analyze", help="build strict success/time/failure/chunk report")
+    ana.add_argument("--run-dir", type=Path, required=True)
+    ana.add_argument("--failure-annotations", type=Path)
+    args = parser.parse_args()
+    cfg = resolve_config(args.config.resolve())
+
+    if args.command == "preflight":
+        report = preflight(cfg)
+        output = args.output or HERE / "results" / f"preflight-{datetime.now().strftime('%Y%m%d-%H%M%S')}.json"
+        dump_json(output, report)
+        print(json.dumps(report, indent=2, ensure_ascii=False))
+        return 0 if report["ready_for_real_validation"] else 2
+    if args.command == "prepare":
+        readiness = preflight(cfg)
+        args.run_dir.mkdir(parents=True, exist_ok=True)
+        dump_json(args.run_dir / "preflight.json", readiness)
+        if not readiness["ready_for_real_validation"]:
+            print(json.dumps(readiness, indent=2, ensure_ascii=False))
+            return 2
+        worktree = prepare_worktree(cfg, args.run_dir.resolve())
+        manifest = write_launch_manifest(cfg, args.run_dir.resolve(), worktree)
+        print(json.dumps(manifest, indent=2, ensure_ascii=False))
+        return 0
+    if args.command == "launch":
+        manifest = load_json(args.run_dir / "launch_manifest.json")
+        selected = list(manifest["arms"]) if args.arm == "all" else [args.arm]
+        execute_arms(manifest, selected)
+        return 0
+    if args.command == "analyze":
+        report = analyze(cfg, args.run_dir.resolve(), args.failure_annotations)
+        print(json.dumps(report, indent=2, ensure_ascii=False))
+        return 0 if report["strict_completion"]["complete"] else 2
+    raise AssertionError(args.command)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())