import hashlib import json from pathlib import Path ROOT = Path(__file__).parent / "validation" def test_persisted_acceptance_status_does_not_claim_unrun_human_audio(): status = json.loads((ROOT / "acceptance_status_2026-07-29.json").read_text()) assert status["safety"]["phone_calls_placed"] == 0 assert status["safety"]["human_audio_captured"] is False assert status["audio_endpoint_probe"]["asr_status"] == "fail" assert status["audio_endpoint_probe"]["asr_error_code"] == "insufficient_quota" assert "no microphone or human audio" in status["audio_endpoint_probe"]["asr_input"] assert status["acceptance_gates"]["authorized_human_participant"]["status"] == "not_run" assert status["acceptance_gates"]["real_human_asr"]["status"] == "not_run" assert status["implementation_gates"]["consent_refusal_before_live_session_construction"]["status"] == "pass_by_test" assert status["implementation_gates"]["barge_in_cancels_playback_and_transcribes"]["status"] == "pass_by_mocked_mechanism_test" assert status["implementation_gates"]["deterministic_judge_and_win_rule"]["status"] == "pass_by_test" assert status["overall_status"] == "incomplete" def test_offline_and_real_llm_evidence_are_explicitly_non_acceptance(): offline = json.loads((ROOT / "offline_privacy_supplement_2026-07-29.json").read_text()) partial = json.loads((ROOT / "real_llm_partial_2026-07-29.json").read_text()) trace = json.loads((ROOT / "real_llm_partial_trace_2026-07-29.json").read_text()) audit = json.loads((ROOT / "real_strategy_audit_supplement_2026-07-29.json").read_text()) assert offline["acceptance_path"] is False assert offline["overall_status"] == "supplemental_only" assert offline["information_isolation_pass"] is True assert partial["acceptance_path"] is False assert partial["gates"]["three_complete_cycles"]["status"] == "fail" assert partial["overall_status"] == "incomplete" assert trace["complete_cycles"] == 2 assert trace["trace_complete"] is False assert any(e.get("action") == "speech" for e in trace["events"]) assert any(e.get("phase") == "vote" for e in trace["events"]) assert audit["acceptance_path"] is False assert audit["human_audio_used"] is False assert audit["audit"]["schema_valid"] is True assert audit["audit"]["overall_pass"] is False assert audit["overall_status"] == "supplemental_only" def test_real_user_simulator_runs_prove_e2e_and_keep_negative_results_separate(): runs = ROOT / "runs" strategy_run = json.loads( (runs / "exp10-6-simulated-user-openrouter-20260801" / "acceptance_report.json").read_text() ) three_cycle_run = json.loads( (runs / "exp10-6-simulated-user-openrouter-20260801-v2" / "acceptance_report.json").read_text() ) strategy_validation = json.loads( (runs / "exp10-6-simulated-user-openrouter-20260801" / "independent_validation.json").read_text() ) three_cycle_validation = json.loads( (runs / "exp10-6-simulated-user-openrouter-20260801-v2" / "independent_validation.json").read_text() ) assert strategy_run["strategy_audit_pass"] is True assert strategy_validation["strict_audio_action_boundary"] == "fail" assert "not an explicit abstention" in strategy_validation["errors"][0] report = three_cycle_run assert report["execution_mode"] == "simulated_user" assert report["acceptance_path"] is True assert report["gates"]["one_llm_user_simulator"]["status"] == "pass" assert report["gates"]["information_isolation"]["status"] == "pass" assert report["gates"]["winner_determined_by_game_rule"]["status"] == "pass" assert report["simulator_llm_tool_calls"] == 2 assert report["simulator_audio_roundtrips"] == 2 ids = [ event.get("response_id") or event.get("request_id") for event in report["voice_events"] if event["type"] in {"simulator_llm_tool", "simulator_asr"} ] assert all(ids) assert len(ids) == len(set(ids)) asr_events = [event for event in report["voice_events"] if event["type"] == "simulator_asr"] assert all( event["usage"]["prompt_tokens_details"]["audio_tokens"] > 0 for event in asr_events ) assert three_cycle_validation["strict_audio_action_boundary"] == "pass" assert three_cycle_run["completed_day_night_vote_cycles"] == 3 assert three_cycle_run["strategy_audit_pass"] is False assert three_cycle_run["overall_status"] == "incomplete" def test_fixed_abstention_has_real_audio_api_receipt(): probe = json.loads((ROOT / "fixed_abstention_probe_20260801.json").read_text()) assert probe["synthetic_audio_only"] is True assert probe["spoken_text"] == "I choose to abstain." assert probe["asr_transcript"] == "I choose to abstain." assert probe["explicit_abstention"] is True assert probe["parsed_target"] is None assert probe["provider"] == "OpenRouter multimodal audio API" assert probe["response_id"].startswith("gen-") assert probe["usage"]["prompt_tokens_details"]["audio_tokens"] > 0 assert len(probe["source_audio_sha256"]) == 64 assert probe["status"] == "pass" def test_latest_formal_run_passes_every_gate_in_one_report(): run = ROOT / "runs" / "exp10-6-simulated-user-openrouter-20260803-v11" report = json.loads((run / "acceptance_report.json").read_text()) independent = json.loads((run / "independent_validation.json").read_text()) assert report["overall_status"] == "pass" assert report["end_to_end_status"] == "pass" assert report["completed_day_night_vote_cycles"] >= 3 assert report["strategy_audit_pass"] is True assert report["information_isolation_pass"] is True assert all( gate["status"] in {"pass", "not_applicable"} for gate in report["gates"].values() ) assert independent["strict_audio_action_boundary"] == "pass" assert independent["source_report_sha256"] == hashlib.sha256( (run / "acceptance_report.json").read_bytes() ).hexdigest() assert independent["simulator_tool_events_checked"] == report["simulator_llm_tool_calls"]