ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
Build latest book artifacts / build (push) Canceled after 0s
dependency resolution / resolve (3.11) (push) Canceled after 0s
dependency resolution / resolve (3.13) (push) Canceled after 0s
deploy-pages / build (push) Canceled after 0s
deploy-pages / deploy (push) Canceled after 0s
i18n consistency check / check (push) Canceled after 0s
provider adoption tests / test (chapter2/context-compression) (push) Canceled after 0s
provider adoption tests / test (chapter2/prompt-injection) (push) Canceled after 0s
provider adoption tests / test (chapter2/system-hint) (push) Canceled after 0s
provider adoption tests / test (chapter3/log-sanitization) (push) Canceled after 0s
web-search-agent tests / test (push) Canceled after 0s
web-search-agent tests / agentbook (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
{
|
||||
"experiment": "6-4",
|
||||
"timestamp_utc": "2026-07-29T10:11:30.432848+00:00",
|
||||
"model": "mlx-community/Qwen2-Audio-7B-Instruct-4bit",
|
||||
"device": "mlx",
|
||||
"method": "growing-prefix full re-encoding (not true streaming)",
|
||||
"cases": [
|
||||
{
|
||||
"scenario": "normal",
|
||||
"audio": "validation/scenarios/normal.wav",
|
||||
"reference": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"qwen2_audio": [
|
||||
{
|
||||
"prefix_seconds": 2.0,
|
||||
"inference_seconds": 4.139199791010469,
|
||||
"transcript": "麻烦你帮我把明天下午的会议",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 4.0,
|
||||
"inference_seconds": 5.053489749785513,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 6.0,
|
||||
"inference_seconds": 4.907809666823596,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 7.4875,
|
||||
"inference_seconds": 5.012733499985188,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': []}"
|
||||
}
|
||||
],
|
||||
"qwen2_final_cer": 0.0,
|
||||
"whisper_vad": {
|
||||
"vad_wait_seconds": 7.4875,
|
||||
"asr_seconds": 0.5402721669524908,
|
||||
"total_response_seconds": 8.027772166952492,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点还是在三号会议时别忘了通知大家",
|
||||
"segment_count": 1,
|
||||
"endpoints": [
|
||||
7.487
|
||||
]
|
||||
},
|
||||
"whisper_final_cer": 0.10526315789473684
|
||||
},
|
||||
{
|
||||
"scenario": "pause",
|
||||
"audio": "validation/scenarios/long_pause.wav",
|
||||
"reference": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"qwen2_audio": [
|
||||
{
|
||||
"prefix_seconds": 2.0,
|
||||
"inference_seconds": 4.358620333019644,
|
||||
"transcript": "麻烦你帮我把明天下午的会议",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 4.0,
|
||||
"inference_seconds": 4.480873542372137,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 6.0,
|
||||
"inference_seconds": 4.751943958923221,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点还还是在三号会议室",
|
||||
"acoustic_events": [
|
||||
"<|noise|>",
|
||||
"<|laughter|>",
|
||||
"<|cough|>"
|
||||
],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点还还是在三号会议室', 'acoustic_events': ['noise', 'laughter', 'cough']}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 8.0,
|
||||
"inference_seconds": 4.8131710002198815,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 8.3875,
|
||||
"inference_seconds": 4.650784750003368,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': []}"
|
||||
}
|
||||
],
|
||||
"qwen2_final_cer": 0.0,
|
||||
"whisper_vad": {
|
||||
"vad_wait_seconds": 4.04,
|
||||
"asr_seconds": 0.5881568747572601,
|
||||
"total_response_seconds": 4.62815687475726,
|
||||
"transcript": "麻煩你幫我把明天下午的會議改到兩點半地點 還是在三號會議時別忘了通知大家",
|
||||
"segment_count": 2,
|
||||
"endpoints": [
|
||||
4.04,
|
||||
8.387
|
||||
]
|
||||
},
|
||||
"whisper_final_cer": 0.42105263157894735
|
||||
},
|
||||
{
|
||||
"scenario": "noise",
|
||||
"audio": "validation/scenarios/background_noise.wav",
|
||||
"reference": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"qwen2_audio": [
|
||||
{
|
||||
"prefix_seconds": 2.0,
|
||||
"inference_seconds": 4.019581000320613,
|
||||
"transcript": "麻烦你帮我把明天下午的会议",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 4.0,
|
||||
"inference_seconds": 4.358317042235285,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点",
|
||||
"acoustic_events": [
|
||||
"<|noise|>",
|
||||
"<|laughter|>",
|
||||
"<|cough|>"
|
||||
],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点', 'acoustic_events': ['noise', 'laughter', 'cough']}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 6.0,
|
||||
"inference_seconds": 4.340823790989816,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 7.4875,
|
||||
"inference_seconds": 4.508679833263159,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [
|
||||
"<|noise|>",
|
||||
"<|laughter|>",
|
||||
"<|cough|>"
|
||||
],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': ['noise', 'laughter', 'cough']}"
|
||||
}
|
||||
],
|
||||
"qwen2_final_cer": 0.0,
|
||||
"whisper_vad": {
|
||||
"vad_wait_seconds": 7.4875,
|
||||
"asr_seconds": 0.4322935417294502,
|
||||
"total_response_seconds": 7.91979354172945,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点还是在三号会议时别忘了通知大家",
|
||||
"segment_count": 1,
|
||||
"endpoints": [
|
||||
7.487
|
||||
]
|
||||
},
|
||||
"whisper_final_cer": 0.10526315789473684
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run_id": "exp6-4-qwen2audio-whisper-provenance-20260730-v3",
|
||||
"run_directory": "validation/runs/exp6-4-qwen2audio-whisper-provenance-20260730-v3",
|
||||
"manifest_sha256": "3d02bce7ac422449bf29bdf61576377b44b9d7b8b046e2cd4f6a7e065f1da67b",
|
||||
"execution_status": "pass",
|
||||
"manuscript_results_reproduced": false
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"experiment": "6-4",
|
||||
"timestamp_utc": "2026-07-29T15:40:54.554752+00:00",
|
||||
"model": "mlx-community/Qwen2-Audio-7B-Instruct-4bit",
|
||||
"device": "mlx",
|
||||
"method": "growing-prefix full re-encoding (not true streaming)",
|
||||
"parameters": {
|
||||
"chunk_seconds": 2.0,
|
||||
"whisper_model": "tiny",
|
||||
"vad_silence_ms": 600,
|
||||
"cer_normalization": "NFKC + Traditional-to-Simplified Chinese + casefold + remove punctuation/symbols/separators"
|
||||
},
|
||||
"provenance": {
|
||||
"host": {
|
||||
"platform": "macOS-26.3-arm64-arm-64bit",
|
||||
"machine": "arm64",
|
||||
"cpu": "Apple M2 Max",
|
||||
"memory_bytes": 103079215104,
|
||||
"python": "3.11.4",
|
||||
"packages": {
|
||||
"mlx-audio": "0.4.6",
|
||||
"openai-whisper": "20231106",
|
||||
"librosa": "0.10.2.post1",
|
||||
"opencc-python-reimplemented": "0.1.7"
|
||||
},
|
||||
"whisper_baseline": {
|
||||
"model": "tiny",
|
||||
"path": "/Users/boj/.cache/whisper/tiny.pt",
|
||||
"sha256": "65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9"
|
||||
}
|
||||
},
|
||||
"qwen2_audio": {
|
||||
"repository": "mlx-community/Qwen2-Audio-7B-Instruct-4bit",
|
||||
"snapshot_revision": "c65570002626f41b4dc08b7b54f42f99f3e82e7f",
|
||||
"snapshot_path": "/Users/boj/.cache/huggingface/hub/models--mlx-community--Qwen2-Audio-7B-Instruct-4bit/snapshots/c65570002626f41b4dc08b7b54f42f99f3e82e7f",
|
||||
"total_bytes": 6574659991,
|
||||
"files": [
|
||||
{
|
||||
"path": ".gitattributes",
|
||||
"size_bytes": 1519,
|
||||
"sha256": "11ad7efa24975ee4b0c3c3a38ed18737f0658a5f75a0a96787b576a78a023361"
|
||||
},
|
||||
{
|
||||
"path": "README.md",
|
||||
"size_bytes": 1716,
|
||||
"sha256": "1cddc760431c9c2c043ed4d17d9c17716a1f81978e6093389420cd46e7f2ce6d"
|
||||
},
|
||||
{
|
||||
"path": "config.json",
|
||||
"size_bytes": 913,
|
||||
"sha256": "39493dad1678533cc2a7ba2de0a573cc926a78ef4c7abef72a6b660159de94b7"
|
||||
},
|
||||
{
|
||||
"path": "merges.txt",
|
||||
"size_bytes": 1671839,
|
||||
"sha256": "599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3"
|
||||
},
|
||||
{
|
||||
"path": "preprocessor_config.json",
|
||||
"size_bytes": 342,
|
||||
"sha256": "4cd7c6c061fe79244c57b0c320b2873f3ee5acce2277f7cc3aced042725680f2"
|
||||
},
|
||||
{
|
||||
"path": "tokenizer.json",
|
||||
"size_bytes": 7028015,
|
||||
"sha256": "f7c9b2dba4a296b1aa76c16a34b8225c0c118978400d4bb66bff0902d702f5b8"
|
||||
},
|
||||
{
|
||||
"path": "tokenizer_config.json",
|
||||
"size_bytes": 638335,
|
||||
"sha256": "c738158e70eeecf25736a6c4d8e5f34cfce683079980757edfb0665a7b2457ed"
|
||||
},
|
||||
{
|
||||
"path": "vocab.json",
|
||||
"size_bytes": 2776833,
|
||||
"sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910"
|
||||
},
|
||||
{
|
||||
"path": "weights.safetensors",
|
||||
"size_bytes": 6562540479,
|
||||
"sha256": null
|
||||
}
|
||||
],
|
||||
"large_weight_hash_note": "Snapshot revision pins large files; files over 10 MiB are inventoried by path and size without rehashing."
|
||||
}
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"scenario": "normal",
|
||||
"audio": "validation/scenarios/normal.wav",
|
||||
"reference": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"media": {
|
||||
"sha256": "22cbbff9df0ec0efc9a09b5d250b8d63c1876363716b8a1d7aa9f85f16e8a410",
|
||||
"expected_acoustic_events": []
|
||||
},
|
||||
"qwen2_audio": [
|
||||
{
|
||||
"prefix_seconds": 2.0,
|
||||
"inference_seconds": 2.859895291738212,
|
||||
"transcript": "麻烦你帮我把明天下午的会议",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 4.0,
|
||||
"inference_seconds": 2.78174329129979,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 6.0,
|
||||
"inference_seconds": 2.8982563340105116,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 7.4875,
|
||||
"inference_seconds": 2.937152124941349,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': []}"
|
||||
}
|
||||
],
|
||||
"qwen2_final_cer": 0.0,
|
||||
"whisper_vad": {
|
||||
"configured_silence_ms": 600,
|
||||
"first_speech_start_seconds": 0.02,
|
||||
"first_endpoint_audio_seconds": 7.16,
|
||||
"first_decision_audio_seconds": 7.76,
|
||||
"post_speech_vad_delay_seconds": 0.6,
|
||||
"first_segment_asr_seconds": 0.5664697499014437,
|
||||
"all_segments_asr_seconds": 0.5664697499014437,
|
||||
"post_endpoint_response_seconds": 1.1664697499014438,
|
||||
"first_response_from_audio_start_seconds": 8.326469749901444,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点还是在三号会议时别忘了通知大家",
|
||||
"segment_transcripts": [
|
||||
"麻烦你帮我把明天下午的会议改到两点半地点还是在三号会议时别忘了通知大家"
|
||||
],
|
||||
"segment_asr_seconds": [
|
||||
0.5664697499014437
|
||||
],
|
||||
"segment_count": 1,
|
||||
"endpoints": [
|
||||
7.16
|
||||
],
|
||||
"decisions": [
|
||||
7.76
|
||||
]
|
||||
},
|
||||
"whisper_final_cer": 0.02857142857142857,
|
||||
"qwen2_event_evaluation": {
|
||||
"true_positive": [],
|
||||
"false_positive": [],
|
||||
"false_negative": [],
|
||||
"exact_match": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "pause",
|
||||
"audio": "validation/scenarios/long_pause.wav",
|
||||
"reference": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"media": {
|
||||
"sha256": "7b4b9975d58c71c50cdb81ff9578a0b04192aeb3cbea634384a804ea4e3dc48d",
|
||||
"expected_acoustic_events": [
|
||||
"<|silence|>"
|
||||
]
|
||||
},
|
||||
"qwen2_audio": [
|
||||
{
|
||||
"prefix_seconds": 2.0,
|
||||
"inference_seconds": 2.6971133751794696,
|
||||
"transcript": "麻烦你帮我把明天下午的会议",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 4.0,
|
||||
"inference_seconds": 2.734142666682601,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 6.0,
|
||||
"inference_seconds": 2.980124874971807,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点还还是在三号会议室",
|
||||
"acoustic_events": [
|
||||
"<|noise|>",
|
||||
"<|laughter|>",
|
||||
"<|cough|>"
|
||||
],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点还还是在三号会议室', 'acoustic_events': ['noise', 'laughter', 'cough']}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 8.0,
|
||||
"inference_seconds": 2.9454508339986205,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 8.3875,
|
||||
"inference_seconds": 2.941487292293459,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': []}"
|
||||
}
|
||||
],
|
||||
"qwen2_final_cer": 0.0,
|
||||
"whisper_vad": {
|
||||
"configured_silence_ms": 600,
|
||||
"first_speech_start_seconds": 0.02,
|
||||
"first_endpoint_audio_seconds": 4.04,
|
||||
"first_decision_audio_seconds": 4.64,
|
||||
"post_speech_vad_delay_seconds": 0.6,
|
||||
"first_segment_asr_seconds": 0.31024629063904285,
|
||||
"all_segments_asr_seconds": 0.5564022487960756,
|
||||
"post_endpoint_response_seconds": 0.9102462906390428,
|
||||
"first_response_from_audio_start_seconds": 4.9502462906390425,
|
||||
"transcript": "麻煩你幫我把明天下午的會議改到兩點半地點 還是在三號會議時別忘了通知大家",
|
||||
"segment_transcripts": [
|
||||
"麻煩你幫我把明天下午的會議改到兩點半地點",
|
||||
"還是在三號會議時別忘了通知大家"
|
||||
],
|
||||
"segment_asr_seconds": [
|
||||
0.31024629063904285,
|
||||
0.24615595815703273
|
||||
],
|
||||
"segment_count": 2,
|
||||
"endpoints": [
|
||||
4.04,
|
||||
8.06
|
||||
],
|
||||
"decisions": [
|
||||
4.64,
|
||||
8.66
|
||||
]
|
||||
},
|
||||
"whisper_final_cer": 0.02857142857142857,
|
||||
"qwen2_event_evaluation": {
|
||||
"true_positive": [],
|
||||
"false_positive": [],
|
||||
"false_negative": [
|
||||
"<|silence|>"
|
||||
],
|
||||
"exact_match": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "noise",
|
||||
"audio": "validation/scenarios/background_noise.wav",
|
||||
"reference": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"media": {
|
||||
"sha256": "034f4e9d4f5c2b69407e1e9813b4544792764a7e2e7340af410945b5c90c48b1",
|
||||
"expected_acoustic_events": [
|
||||
"<|noise|>"
|
||||
]
|
||||
},
|
||||
"qwen2_audio": [
|
||||
{
|
||||
"prefix_seconds": 2.0,
|
||||
"inference_seconds": 2.711420916952193,
|
||||
"transcript": "麻烦你帮我把明天下午的会议",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 4.0,
|
||||
"inference_seconds": 2.9726402079686522,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点",
|
||||
"acoustic_events": [
|
||||
"<|noise|>",
|
||||
"<|laughter|>",
|
||||
"<|cough|>"
|
||||
],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点', 'acoustic_events': ['noise', 'laughter', 'cough']}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 6.0,
|
||||
"inference_seconds": 3.041755000129342,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 7.4875,
|
||||
"inference_seconds": 3.2653073337860405,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [
|
||||
"<|noise|>",
|
||||
"<|laughter|>",
|
||||
"<|cough|>"
|
||||
],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': ['noise', 'laughter', 'cough']}"
|
||||
}
|
||||
],
|
||||
"qwen2_final_cer": 0.0,
|
||||
"whisper_vad": {
|
||||
"configured_silence_ms": 600,
|
||||
"first_speech_start_seconds": 0.0,
|
||||
"first_endpoint_audio_seconds": 7.4875,
|
||||
"first_decision_audio_seconds": 8.0875,
|
||||
"post_speech_vad_delay_seconds": 0.6,
|
||||
"first_segment_asr_seconds": 0.40538191702216864,
|
||||
"all_segments_asr_seconds": 0.40538191702216864,
|
||||
"post_endpoint_response_seconds": 1.0053819170221687,
|
||||
"first_response_from_audio_start_seconds": 8.492881917022169,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点还是在三号会议时别忘了通知大家",
|
||||
"segment_transcripts": [
|
||||
"麻烦你帮我把明天下午的会议改到两点半地点还是在三号会议时别忘了通知大家"
|
||||
],
|
||||
"segment_asr_seconds": [
|
||||
0.40538191702216864
|
||||
],
|
||||
"segment_count": 1,
|
||||
"endpoints": [
|
||||
7.487
|
||||
],
|
||||
"decisions": [
|
||||
8.088
|
||||
]
|
||||
},
|
||||
"whisper_final_cer": 0.02857142857142857,
|
||||
"qwen2_event_evaluation": {
|
||||
"true_positive": [
|
||||
"<|noise|>"
|
||||
],
|
||||
"false_positive": [
|
||||
"<|cough|>",
|
||||
"<|laughter|>"
|
||||
],
|
||||
"false_negative": [],
|
||||
"exact_match": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"cost": {
|
||||
"paid_external_requests": 0,
|
||||
"total_usd": 0,
|
||||
"note": "Both models ran locally."
|
||||
},
|
||||
"acceptance": {
|
||||
"execution_gates": {
|
||||
"real_qwen2_audio": true,
|
||||
"growing_prefix_full_reencoding": true,
|
||||
"real_600ms_vad_whisper_baseline": true,
|
||||
"normal_pause_noise_scenarios": true,
|
||||
"corrected_vad_latency_accounting": true,
|
||||
"normalized_cer": true,
|
||||
"provenance_complete": true
|
||||
},
|
||||
"execution_passed": true,
|
||||
"manuscript_result_claims": {
|
||||
"qwen_incremental_latency_100_to_200ms": false,
|
||||
"traditional_post_endpoint_latency_800_to_1100ms": false,
|
||||
"pause_split_into_two_segments": true,
|
||||
"pause_specific_two_to_zero_error": false,
|
||||
"noise_token_detected": true,
|
||||
"noise_caused_earlier_vad_start": false
|
||||
},
|
||||
"manuscript_results_reproduced": false
|
||||
}
|
||||
}
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"experiment": "6-4",
|
||||
"run_id": "exp6-4-qwen2audio-whisper-provenance-20260730-v3",
|
||||
"started_at": "2026-07-30T05:22:28.357Z",
|
||||
"completed_at": "2026-07-30T05:24:59.451Z",
|
||||
"duration_seconds": 151.093,
|
||||
"git_commit": "f119cf510f3746cd5d071d0bce0051617f02f163",
|
||||
"execution_gates": {
|
||||
"exact_qwen2audio_mlx_design": {
|
||||
"status": "pass"
|
||||
},
|
||||
"exact_600ms_vad_whisper_tiny_design": {
|
||||
"status": "pass"
|
||||
},
|
||||
"normal_pause_noise_inputs": {
|
||||
"status": "pass"
|
||||
},
|
||||
"raw_qwen_outputs_retained": {
|
||||
"status": "pass",
|
||||
"prefix_count": 13
|
||||
},
|
||||
"runtime_sources_stable_and_hashed": {
|
||||
"status": "pass",
|
||||
"count": 5
|
||||
},
|
||||
"full_model_snapshot_hashed": {
|
||||
"status": "pass",
|
||||
"revision": "c65570002626f41b4dc08b7b54f42f99f3e82e7f",
|
||||
"file_count": 9,
|
||||
"total_bytes": 6574659991
|
||||
},
|
||||
"whisper_checkpoint_hashed": {
|
||||
"status": "pass",
|
||||
"size_bytes": 75572083
|
||||
},
|
||||
"credential_free_local_artifacts": {
|
||||
"status": "pass",
|
||||
"actual_secret_hits": 0,
|
||||
"credential_pattern_hits": 0
|
||||
}
|
||||
},
|
||||
"execution_status": "pass",
|
||||
"manuscript_result_claims": {
|
||||
"qwen_incremental_latency_100_to_200ms": false,
|
||||
"traditional_post_endpoint_latency_800_to_1100ms": false,
|
||||
"pause_split_into_two_segments": true,
|
||||
"pause_specific_two_to_zero_error": false,
|
||||
"noise_token_detected": true,
|
||||
"noise_caused_earlier_vad_start": false
|
||||
},
|
||||
"manuscript_results_reproduced": false,
|
||||
"event_evaluation": {
|
||||
"normal": {
|
||||
"true_positive": [],
|
||||
"false_positive": [],
|
||||
"false_negative": [],
|
||||
"exact_match": true
|
||||
},
|
||||
"pause": {
|
||||
"true_positive": [],
|
||||
"false_positive": [],
|
||||
"false_negative": [
|
||||
"<|silence|>"
|
||||
],
|
||||
"exact_match": false
|
||||
},
|
||||
"noise": {
|
||||
"true_positive": [
|
||||
"<|noise|>"
|
||||
],
|
||||
"false_positive": [
|
||||
"<|cough|>",
|
||||
"<|laughter|>"
|
||||
],
|
||||
"false_negative": [],
|
||||
"exact_match": false
|
||||
}
|
||||
},
|
||||
"qwen_inference_seconds": [
|
||||
9.968843291979283,
|
||||
9.877914333250374,
|
||||
10.44804058270529,
|
||||
10.78756216680631,
|
||||
8.413955624680966,
|
||||
8.742833625059575,
|
||||
10.989260375034064,
|
||||
10.69584474992007,
|
||||
10.8182367910631,
|
||||
8.962713832966983,
|
||||
10.234860499855131,
|
||||
10.288799332920462,
|
||||
11.268820082768798
|
||||
],
|
||||
"whisper_post_endpoint_response_seconds": {
|
||||
"normal": 1.2523482501506806,
|
||||
"pause": 0.9871377082541585,
|
||||
"noise": 1.155937457829714
|
||||
},
|
||||
"runtime_source_sha256": {
|
||||
"run_official_experiment.py": "a1d95ed2fbd9405678a146898201c7e1082fd2620f6528268a7ee8943f22e5b1",
|
||||
"demo.py": "bf132207426606a38c4e487ed59ff7832cfa2468f5c6e2bab87394086a049d92",
|
||||
"qwen2_streaming.py": "7744acd4c134c9f861d207ff400d5effa32d17d400ca4973da589d8d74782c76",
|
||||
"whisper_baseline.py": "dc0554b4085e806bc18e8a81fe5c0af55d34bfbe43107b66632afd461c7b5483",
|
||||
"prepare_scenarios.py": "a0b01392d19a4c0463d33c7b56af323131cf7a42213410537109a6698f1e15b0"
|
||||
},
|
||||
"audio_sha256": {
|
||||
"audio/sentence.wav": "51b77857cb9d859ca37dc6ba95680fb675b8e2690257f00f69a46a11febbd4a2",
|
||||
"validation/scenarios/normal.wav": "22cbbff9df0ec0efc9a09b5d250b8d63c1876363716b8a1d7aa9f85f16e8a410",
|
||||
"validation/scenarios/long_pause.wav": "7b4b9975d58c71c50cdb81ff9578a0b04192aeb3cbea634384a804ea4e3dc48d",
|
||||
"validation/scenarios/background_noise.wav": "034f4e9d4f5c2b69407e1e9813b4544792764a7e2e7340af410945b5c90c48b1"
|
||||
},
|
||||
"whisper_checkpoint": {
|
||||
"path": "/Users/boj/.cache/whisper/tiny.pt",
|
||||
"size_bytes": 75572083,
|
||||
"sha256": "65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9"
|
||||
},
|
||||
"qwen2_audio_snapshot": {
|
||||
"repository": "mlx-community/Qwen2-Audio-7B-Instruct-4bit",
|
||||
"revision": "c65570002626f41b4dc08b7b54f42f99f3e82e7f",
|
||||
"path": "/Users/boj/.cache/huggingface/hub/models--mlx-community--Qwen2-Audio-7B-Instruct-4bit/snapshots/c65570002626f41b4dc08b7b54f42f99f3e82e7f",
|
||||
"files": {
|
||||
".gitattributes": {
|
||||
"size_bytes": 1519,
|
||||
"sha256": "11ad7efa24975ee4b0c3c3a38ed18737f0658a5f75a0a96787b576a78a023361"
|
||||
},
|
||||
"README.md": {
|
||||
"size_bytes": 1716,
|
||||
"sha256": "1cddc760431c9c2c043ed4d17d9c17716a1f81978e6093389420cd46e7f2ce6d"
|
||||
},
|
||||
"config.json": {
|
||||
"size_bytes": 913,
|
||||
"sha256": "39493dad1678533cc2a7ba2de0a573cc926a78ef4c7abef72a6b660159de94b7"
|
||||
},
|
||||
"merges.txt": {
|
||||
"size_bytes": 1671839,
|
||||
"sha256": "599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3"
|
||||
},
|
||||
"preprocessor_config.json": {
|
||||
"size_bytes": 342,
|
||||
"sha256": "4cd7c6c061fe79244c57b0c320b2873f3ee5acce2277f7cc3aced042725680f2"
|
||||
},
|
||||
"tokenizer.json": {
|
||||
"size_bytes": 7028015,
|
||||
"sha256": "f7c9b2dba4a296b1aa76c16a34b8225c0c118978400d4bb66bff0902d702f5b8"
|
||||
},
|
||||
"tokenizer_config.json": {
|
||||
"size_bytes": 638335,
|
||||
"sha256": "c738158e70eeecf25736a6c4d8e5f34cfce683079980757edfb0665a7b2457ed"
|
||||
},
|
||||
"vocab.json": {
|
||||
"size_bytes": 2776833,
|
||||
"sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910"
|
||||
},
|
||||
"weights.safetensors": {
|
||||
"size_bytes": 6562540479,
|
||||
"sha256": "0967cde270ad62aa4824f0bbce283d0a2a6da2825ccf587640753c527d4174da"
|
||||
}
|
||||
}
|
||||
},
|
||||
"credential_scan": {
|
||||
"actual_secret_hits": 0,
|
||||
"credential_pattern_hits": 0
|
||||
},
|
||||
"passed_gates": 8,
|
||||
"total_gates": 8
|
||||
}
|
||||
+366
@@ -0,0 +1,366 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"experiment": "6-4",
|
||||
"timestamp_utc": "2026-07-30T05:24:57.619611+00:00",
|
||||
"model": "mlx-community/Qwen2-Audio-7B-Instruct-4bit",
|
||||
"device": "mlx",
|
||||
"method": "growing-prefix full re-encoding (not true streaming)",
|
||||
"parameters": {
|
||||
"chunk_seconds": 2.0,
|
||||
"whisper_model": "tiny",
|
||||
"vad_silence_ms": 600,
|
||||
"cer_normalization": "NFKC + Traditional-to-Simplified Chinese + casefold + remove punctuation/symbols/separators"
|
||||
},
|
||||
"provenance": {
|
||||
"host": {
|
||||
"platform": "macOS-26.3-arm64-arm-64bit",
|
||||
"machine": "arm64",
|
||||
"cpu": "Apple M2 Max",
|
||||
"memory_bytes": 103079215104,
|
||||
"python": "3.11.4",
|
||||
"packages": {
|
||||
"mlx-audio": "0.4.6",
|
||||
"openai-whisper": "20231106",
|
||||
"librosa": "0.10.2.post1",
|
||||
"opencc-python-reimplemented": "0.1.7"
|
||||
},
|
||||
"whisper_baseline": {
|
||||
"model": "tiny",
|
||||
"path": "/Users/boj/.cache/whisper/tiny.pt",
|
||||
"sha256": "65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9"
|
||||
}
|
||||
},
|
||||
"qwen2_audio": {
|
||||
"repository": "mlx-community/Qwen2-Audio-7B-Instruct-4bit",
|
||||
"snapshot_revision": "c65570002626f41b4dc08b7b54f42f99f3e82e7f",
|
||||
"snapshot_path": "/Users/boj/.cache/huggingface/hub/models--mlx-community--Qwen2-Audio-7B-Instruct-4bit/snapshots/c65570002626f41b4dc08b7b54f42f99f3e82e7f",
|
||||
"total_bytes": 6574659991,
|
||||
"files": [
|
||||
{
|
||||
"path": ".gitattributes",
|
||||
"size_bytes": 1519,
|
||||
"sha256": "11ad7efa24975ee4b0c3c3a38ed18737f0658a5f75a0a96787b576a78a023361"
|
||||
},
|
||||
{
|
||||
"path": "README.md",
|
||||
"size_bytes": 1716,
|
||||
"sha256": "1cddc760431c9c2c043ed4d17d9c17716a1f81978e6093389420cd46e7f2ce6d"
|
||||
},
|
||||
{
|
||||
"path": "config.json",
|
||||
"size_bytes": 913,
|
||||
"sha256": "39493dad1678533cc2a7ba2de0a573cc926a78ef4c7abef72a6b660159de94b7"
|
||||
},
|
||||
{
|
||||
"path": "merges.txt",
|
||||
"size_bytes": 1671839,
|
||||
"sha256": "599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3"
|
||||
},
|
||||
{
|
||||
"path": "preprocessor_config.json",
|
||||
"size_bytes": 342,
|
||||
"sha256": "4cd7c6c061fe79244c57b0c320b2873f3ee5acce2277f7cc3aced042725680f2"
|
||||
},
|
||||
{
|
||||
"path": "tokenizer.json",
|
||||
"size_bytes": 7028015,
|
||||
"sha256": "f7c9b2dba4a296b1aa76c16a34b8225c0c118978400d4bb66bff0902d702f5b8"
|
||||
},
|
||||
{
|
||||
"path": "tokenizer_config.json",
|
||||
"size_bytes": 638335,
|
||||
"sha256": "c738158e70eeecf25736a6c4d8e5f34cfce683079980757edfb0665a7b2457ed"
|
||||
},
|
||||
{
|
||||
"path": "vocab.json",
|
||||
"size_bytes": 2776833,
|
||||
"sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910"
|
||||
},
|
||||
{
|
||||
"path": "weights.safetensors",
|
||||
"size_bytes": 6562540479,
|
||||
"sha256": null
|
||||
}
|
||||
],
|
||||
"large_weight_hash_note": "Snapshot revision pins large files; files over 10 MiB are inventoried by path and size without rehashing."
|
||||
}
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"scenario": "normal",
|
||||
"audio": "validation/scenarios/normal.wav",
|
||||
"reference": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"media": {
|
||||
"sha256": "22cbbff9df0ec0efc9a09b5d250b8d63c1876363716b8a1d7aa9f85f16e8a410",
|
||||
"expected_acoustic_events": []
|
||||
},
|
||||
"qwen2_audio": [
|
||||
{
|
||||
"prefix_seconds": 2.0,
|
||||
"inference_seconds": 9.968843291979283,
|
||||
"transcript": "麻烦你帮我把明天下午的会议",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 4.0,
|
||||
"inference_seconds": 9.877914333250374,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 6.0,
|
||||
"inference_seconds": 10.44804058270529,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 7.4875,
|
||||
"inference_seconds": 10.78756216680631,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': []}"
|
||||
}
|
||||
],
|
||||
"qwen2_final_cer": 0.0,
|
||||
"whisper_vad": {
|
||||
"configured_silence_ms": 600,
|
||||
"first_speech_start_seconds": 0.02,
|
||||
"first_endpoint_audio_seconds": 7.16,
|
||||
"first_decision_audio_seconds": 7.76,
|
||||
"post_speech_vad_delay_seconds": 0.6,
|
||||
"first_segment_asr_seconds": 0.6523482501506805,
|
||||
"all_segments_asr_seconds": 0.6523482501506805,
|
||||
"post_endpoint_response_seconds": 1.2523482501506806,
|
||||
"first_response_from_audio_start_seconds": 8.41234825015068,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点还是在三号会议时别忘了通知大家",
|
||||
"segment_transcripts": [
|
||||
"麻烦你帮我把明天下午的会议改到两点半地点还是在三号会议时别忘了通知大家"
|
||||
],
|
||||
"segment_asr_seconds": [
|
||||
0.6523482501506805
|
||||
],
|
||||
"segment_count": 1,
|
||||
"endpoints": [
|
||||
7.16
|
||||
],
|
||||
"decisions": [
|
||||
7.76
|
||||
]
|
||||
},
|
||||
"whisper_final_cer": 0.02857142857142857,
|
||||
"qwen2_event_evaluation": {
|
||||
"true_positive": [],
|
||||
"false_positive": [],
|
||||
"false_negative": [],
|
||||
"exact_match": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "pause",
|
||||
"audio": "validation/scenarios/long_pause.wav",
|
||||
"reference": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"media": {
|
||||
"sha256": "7b4b9975d58c71c50cdb81ff9578a0b04192aeb3cbea634384a804ea4e3dc48d",
|
||||
"expected_acoustic_events": [
|
||||
"<|silence|>"
|
||||
]
|
||||
},
|
||||
"qwen2_audio": [
|
||||
{
|
||||
"prefix_seconds": 2.0,
|
||||
"inference_seconds": 8.413955624680966,
|
||||
"transcript": "麻烦你帮我把明天下午的会议",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 4.0,
|
||||
"inference_seconds": 8.742833625059575,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 6.0,
|
||||
"inference_seconds": 10.989260375034064,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点还还是在三号会议室",
|
||||
"acoustic_events": [
|
||||
"<|noise|>",
|
||||
"<|laughter|>",
|
||||
"<|cough|>"
|
||||
],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点还还是在三号会议室', 'acoustic_events': ['noise', 'laughter', 'cough']}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 8.0,
|
||||
"inference_seconds": 10.69584474992007,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 8.3875,
|
||||
"inference_seconds": 10.8182367910631,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': []}"
|
||||
}
|
||||
],
|
||||
"qwen2_final_cer": 0.0,
|
||||
"whisper_vad": {
|
||||
"configured_silence_ms": 600,
|
||||
"first_speech_start_seconds": 0.02,
|
||||
"first_endpoint_audio_seconds": 4.04,
|
||||
"first_decision_audio_seconds": 4.64,
|
||||
"post_speech_vad_delay_seconds": 0.6,
|
||||
"first_segment_asr_seconds": 0.3871377082541585,
|
||||
"all_segments_asr_seconds": 0.7301125833764672,
|
||||
"post_endpoint_response_seconds": 0.9871377082541585,
|
||||
"first_response_from_audio_start_seconds": 5.027137708254158,
|
||||
"transcript": "麻煩你幫我把明天下午的會議改到兩點半地點 還是在三號會議時別忘了通知大家",
|
||||
"segment_transcripts": [
|
||||
"麻煩你幫我把明天下午的會議改到兩點半地點",
|
||||
"還是在三號會議時別忘了通知大家"
|
||||
],
|
||||
"segment_asr_seconds": [
|
||||
0.3871377082541585,
|
||||
0.34297487512230873
|
||||
],
|
||||
"segment_count": 2,
|
||||
"endpoints": [
|
||||
4.04,
|
||||
8.06
|
||||
],
|
||||
"decisions": [
|
||||
4.64,
|
||||
8.66
|
||||
]
|
||||
},
|
||||
"whisper_final_cer": 0.02857142857142857,
|
||||
"qwen2_event_evaluation": {
|
||||
"true_positive": [],
|
||||
"false_positive": [],
|
||||
"false_negative": [
|
||||
"<|silence|>"
|
||||
],
|
||||
"exact_match": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "noise",
|
||||
"audio": "validation/scenarios/background_noise.wav",
|
||||
"reference": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"media": {
|
||||
"sha256": "034f4e9d4f5c2b69407e1e9813b4544792764a7e2e7340af410945b5c90c48b1",
|
||||
"expected_acoustic_events": [
|
||||
"<|noise|>"
|
||||
]
|
||||
},
|
||||
"qwen2_audio": [
|
||||
{
|
||||
"prefix_seconds": 2.0,
|
||||
"inference_seconds": 8.962713832966983,
|
||||
"transcript": "麻烦你帮我把明天下午的会议",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 4.0,
|
||||
"inference_seconds": 10.234860499855131,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点",
|
||||
"acoustic_events": [
|
||||
"<|noise|>",
|
||||
"<|laughter|>",
|
||||
"<|cough|>"
|
||||
],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半地点', 'acoustic_events': ['noise', 'laughter', 'cough']}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 6.0,
|
||||
"inference_seconds": 10.288799332920462,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。",
|
||||
"acoustic_events": [],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室。', 'acoustic_events': []}"
|
||||
},
|
||||
{
|
||||
"prefix_seconds": 7.4875,
|
||||
"inference_seconds": 11.268820082768798,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。",
|
||||
"acoustic_events": [
|
||||
"<|noise|>",
|
||||
"<|laughter|>",
|
||||
"<|cough|>"
|
||||
],
|
||||
"raw_response": "{'transcript': '麻烦你帮我把明天下午的会议改到两点半，地点还是在三号会议室，别忘了通知大家。', 'acoustic_events': ['noise', 'laughter', 'cough']}"
|
||||
}
|
||||
],
|
||||
"qwen2_final_cer": 0.0,
|
||||
"whisper_vad": {
|
||||
"configured_silence_ms": 600,
|
||||
"first_speech_start_seconds": 0.0,
|
||||
"first_endpoint_audio_seconds": 7.4875,
|
||||
"first_decision_audio_seconds": 8.0875,
|
||||
"post_speech_vad_delay_seconds": 0.6,
|
||||
"first_segment_asr_seconds": 0.5559374578297138,
|
||||
"all_segments_asr_seconds": 0.5559374578297138,
|
||||
"post_endpoint_response_seconds": 1.155937457829714,
|
||||
"first_response_from_audio_start_seconds": 8.643437457829714,
|
||||
"transcript": "麻烦你帮我把明天下午的会议改到两点半地点还是在三号会议时别忘了通知大家",
|
||||
"segment_transcripts": [
|
||||
"麻烦你帮我把明天下午的会议改到两点半地点还是在三号会议时别忘了通知大家"
|
||||
],
|
||||
"segment_asr_seconds": [
|
||||
0.5559374578297138
|
||||
],
|
||||
"segment_count": 1,
|
||||
"endpoints": [
|
||||
7.487
|
||||
],
|
||||
"decisions": [
|
||||
8.088
|
||||
]
|
||||
},
|
||||
"whisper_final_cer": 0.02857142857142857,
|
||||
"qwen2_event_evaluation": {
|
||||
"true_positive": [
|
||||
"<|noise|>"
|
||||
],
|
||||
"false_positive": [
|
||||
"<|cough|>",
|
||||
"<|laughter|>"
|
||||
],
|
||||
"false_negative": [],
|
||||
"exact_match": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"cost": {
|
||||
"paid_external_requests": 0,
|
||||
"total_usd": 0,
|
||||
"note": "Both models ran locally."
|
||||
},
|
||||
"acceptance": {
|
||||
"execution_gates": {
|
||||
"real_qwen2_audio": true,
|
||||
"growing_prefix_full_reencoding": true,
|
||||
"real_600ms_vad_whisper_baseline": true,
|
||||
"normal_pause_noise_scenarios": true,
|
||||
"corrected_vad_latency_accounting": true,
|
||||
"normalized_cer": true,
|
||||
"provenance_complete": true
|
||||
},
|
||||
"execution_passed": true,
|
||||
"manuscript_result_claims": {
|
||||
"qwen_incremental_latency_100_to_200ms": false,
|
||||
"traditional_post_endpoint_latency_800_to_1100ms": false,
|
||||
"pause_split_into_two_segments": true,
|
||||
"pause_specific_two_to_zero_error": false,
|
||||
"noise_token_detected": true,
|
||||
"noise_caused_earlier_vad_start": false
|
||||
},
|
||||
"manuscript_results_reproduced": false
|
||||
}
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"experiment": "6-4",
|
||||
"run_id": "exp6-4-qwen2audio-whisper-provenance-20260730-v3",
|
||||
"generated_at": "2026-07-30T05:24:59.463Z",
|
||||
"git_commit": "f119cf510f3746cd5d071d0bce0051617f02f163",
|
||||
"runtime_source_sha256": {
|
||||
"run_official_experiment.py": "caef17357a38fb3d5215c9d3108a933da8f219751936c6a9bb70411655037a22",
|
||||
"demo.py": "b5f8169870d55627410975f601b8ac60afa25cd47dcf7e01376923c4dd503379",
|
||||
"qwen2_streaming.py": "cac5aad7b578c7033a1becc3abb91cfe094623d5a99a9ccc4362cc2b094b7ead",
|
||||
"whisper_baseline.py": "4547b92fe78001e0b093e43d0f51669a197fb0411bfea06a2e5a9f82e84bc245",
|
||||
"prepare_scenarios.py": "a0b01392d19a4c0463d33c7b56af323131cf7a42213410537109a6698f1e15b0"
|
||||
},
|
||||
"audio_sha256": {
|
||||
"audio/sentence.wav": "51b77857cb9d859ca37dc6ba95680fb675b8e2690257f00f69a46a11febbd4a2",
|
||||
"validation/scenarios/normal.wav": "22cbbff9df0ec0efc9a09b5d250b8d63c1876363716b8a1d7aa9f85f16e8a410",
|
||||
"validation/scenarios/long_pause.wav": "7b4b9975d58c71c50cdb81ff9578a0b04192aeb3cbea634384a804ea4e3dc48d",
|
||||
"validation/scenarios/background_noise.wav": "034f4e9d4f5c2b69407e1e9813b4544792764a7e2e7340af410945b5c90c48b1"
|
||||
},
|
||||
"whisper_checkpoint": {
|
||||
"path": "/Users/boj/.cache/whisper/tiny.pt",
|
||||
"size_bytes": 75572083,
|
||||
"sha256": "65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9"
|
||||
},
|
||||
"qwen2_audio_snapshot": {
|
||||
"repository": "mlx-community/Qwen2-Audio-7B-Instruct-4bit",
|
||||
"revision": "c65570002626f41b4dc08b7b54f42f99f3e82e7f",
|
||||
"path": "/Users/boj/.cache/huggingface/hub/models--mlx-community--Qwen2-Audio-7B-Instruct-4bit/snapshots/c65570002626f41b4dc08b7b54f42f99f3e82e7f",
|
||||
"files": {
|
||||
".gitattributes": {
|
||||
"size_bytes": 1519,
|
||||
"sha256": "11ad7efa24975ee4b0c3c3a38ed18737f0658a5f75a0a96787b576a78a023361"
|
||||
},
|
||||
"README.md": {
|
||||
"size_bytes": 1716,
|
||||
"sha256": "1cddc760431c9c2c043ed4d17d9c17716a1f81978e6093389420cd46e7f2ce6d"
|
||||
},
|
||||
"config.json": {
|
||||
"size_bytes": 913,
|
||||
"sha256": "39493dad1678533cc2a7ba2de0a573cc926a78ef4c7abef72a6b660159de94b7"
|
||||
},
|
||||
"merges.txt": {
|
||||
"size_bytes": 1671839,
|
||||
"sha256": "599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3"
|
||||
},
|
||||
"preprocessor_config.json": {
|
||||
"size_bytes": 342,
|
||||
"sha256": "4cd7c6c061fe79244c57b0c320b2873f3ee5acce2277f7cc3aced042725680f2"
|
||||
},
|
||||
"tokenizer.json": {
|
||||
"size_bytes": 7028015,
|
||||
"sha256": "f7c9b2dba4a296b1aa76c16a34b8225c0c118978400d4bb66bff0902d702f5b8"
|
||||
},
|
||||
"tokenizer_config.json": {
|
||||
"size_bytes": 638335,
|
||||
"sha256": "c738158e70eeecf25736a6c4d8e5f34cfce683079980757edfb0665a7b2457ed"
|
||||
},
|
||||
"vocab.json": {
|
||||
"size_bytes": 2776833,
|
||||
"sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910"
|
||||
},
|
||||
"weights.safetensors": {
|
||||
"size_bytes": 6562540479,
|
||||
"sha256": "0967cde270ad62aa4824f0bbce283d0a2a6da2825ccf587640753c527d4174da"
|
||||
}
|
||||
}
|
||||
},
|
||||
"artifact_sha256": {
|
||||
"evidence.json": "17d5fa7cffbab6d2ae196008a4584e3f37d41cb6af541d0ffbcf184c3711c057",
|
||||
"run.log": "d1b29fffbf023ad11535277b209f11520318e2f73f1e7f2f2c6d3da8901bed2d",
|
||||
"acceptance.json": "ef593df76bb751ca311f4fdc50471210309e45b58aa4b1a5d52412823c40eefb"
|
||||
},
|
||||
"acceptance": {
|
||||
"execution_status": "pass",
|
||||
"passed_gates": 8,
|
||||
"total_gates": 8,
|
||||
"manuscript_results_reproduced": false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user