#!/usr/bin/env python3
"""Serve an HTML report visualizing experiment arms from validation/latest.json.
Reads validation/latest.json (next to this script), then serves a report over
HTTP and opens it in the browser. The report shows:
1. A summary table with task_success / iterations / repeated_tool_calls per arm.
2. One section per arm showing tool_call_signatures and reasoning_steps,
with every reasoning step collapsed by default.
"""
import argparse
import html
import json
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
DEFAULT_JSON = SCRIPT_DIR / "validation" / "latest.json"
def esc(value):
return html.escape(str(value))
def fmt(value):
if value is None:
return "—"
if isinstance(value, bool):
return str(value)
return html.escape(str(value))
def status_badge(value):
if value is True:
return '✓'
if value is False:
return '✗'
return 'n/a'
def render_arm_section(arm):
mode = esc(arm.get("mode", "?"))
model = esc(arm.get("model", ""))
provider = esc(arm.get("provider", ""))
elapsed = arm.get("elapsed_seconds")
elapsed_s = f"{elapsed:.2f} s" if isinstance(elapsed, (int, float)) else "—"
final_answer = arm.get("final_answer")
parts = [f'']
parts.append(f"Mode: {mode}
")
parts.append(
f'{provider} / {model} · {elapsed_s} '
f'· completed={fmt(arm.get("completed"))} '
f'success={fmt(arm.get("success"))}
'
)
if final_answer:
parts.append(
f'Final answer
'
)
signatures = arm.get("tool_call_signatures") or []
parts.append(f"Tool call signatures ({len(signatures)})
")
if signatures:
parts.append('')
for i, sig in enumerate(signatures, start=1):
parts.append(
f'- {i}'
f'
{esc(sig)} '
)
parts.append("
")
else:
parts.append("No tool calls.
")
reasoning = arm.get("reasoning_steps") or []
parts.append(f"Reasoning steps ({len(reasoning)})
")
if reasoning:
parts.append('')
for i, step in enumerate(reasoning, start=1):
if isinstance(step, str):
body = esc(step)
else:
body = esc(json.dumps(step, ensure_ascii=False, indent=2))
parts.append(
f"
"
f"Step {i}
"
f"{body}"
f" "
)
parts.append("
No reasoning steps recorded.
")
parts.append("")
return "\n".join(parts)
def render(data):
arms = data.get("arms", [])
expected = data.get("expected_numbers", [])
rows = []
for arm in arms:
behavior = arm.get("behavior") or {}
elapsed = arm.get("elapsed_seconds")
elapsed_s = f"{elapsed:.2f}" if isinstance(elapsed, (int, float)) else "—"
rows.append(
"".join(
[
"",
f'| {esc(arm.get("mode", "?"))} | ',
f"{status_badge(arm.get('task_success'))} | ",
f"{fmt(arm.get('iterations'))} | ",
f"{fmt(arm.get('repeated_tool_calls'))} | ",
f"{fmt(behavior.get('tool_action_count'))} | ",
f"{elapsed_s} | ",
"
",
]
)
)
table = (
""
""
"| mode | "
"task_success | "
"iterations | "
"repeated_tool_calls | "
"tool actions | "
"elapsed (s) | "
"
"
f"{''.join(rows)}"
"
"
)
sections = "\n".join(render_arm_section(arm) for arm in arms)
expected_html = ""
if expected:
items = "".join(f"{esc(n)}" for n in expected)
expected_html = (
'Expected numbers
'
f''
)
return f"""
Experiment {esc(data.get('experiment_id', ''))} — arm comparison
Experiment {esc(data.get('experiment_id', ''))}
source: {esc(data.get('canonical_source', ''))} · created: {esc(data.get('created_at', ''))}
Task
{esc(data.get('task', ''))}
Summary
{table}
{sections}
"""
def serve(data, port):
import http.server
import threading
import webbrowser
html_body = render(data)
url = f"http://127.0.0.1:{port}"
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
if self.path not in ("/", "/index.html"):
self.send_error(404)
return
body = html_body.encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, fmt, *args):
print("[http] %s - %s" % (self.address_string(), fmt % args))
server = http.server.HTTPServer(("127.0.0.1", port), Handler)
print(f"Serving on {url} (Ctrl+C to stop)")
threading.Timer(0.5, lambda: webbrowser.open(url)).start()
try:
server.serve_forever()
except KeyboardInterrupt:
print("\nStopped.")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("json_path", nargs="?", default=str(DEFAULT_JSON),
help="Path to the validation JSON (default: %(default)s)")
parser.add_argument("-p", "--port", type=int, default=8000,
help="Port to serve on (default: 8000)")
args = parser.parse_args()
json_file = Path(args.json_path)
with open(json_file, encoding="utf-8") as fh:
data = json.load(fh)
serve(data, args.port)
if __name__ == "__main__":
main()