tiny-voca-ar / evaluation.json
abdullah's picture
Release v0.1.0
bcf6cf1 verified
Raw History Blame Contribute Delete
5.91 kB
{
"student": {
"cases": 43,
"mean_wer": 0.0532,
"judge1_vowels": 1375,
"judge1_doubtful": 12,
"judge1_agreement": 0.9913,
"digits_expected": 67,
"digits_correct": 67,
"kinds": {
"short": {
"cases": 3,
"mean_wer": 0.0,
"doubtful_vowels": 0,
"vowels": 25,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 12.7
},
"medium": {
"cases": 3,
"mean_wer": 0.0,
"doubtful_vowels": 0,
"vowels": 89,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 651.5
},
"long": {
"cases": 3,
"mean_wer": 0.0074,
"doubtful_vowels": 0,
"vowels": 306,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 670.6
},
"numbers": {
"cases": 3,
"mean_wer": 0.0,
"doubtful_vowels": 2,
"vowels": 132,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 747.6
},
"phone": {
"cases": 5,
"mean_wer": 0.0544,
"doubtful_vowels": 4,
"vowels": 169,
"unread_words": 0,
"digit_accuracy": 1.0,
"times_real_time": 652.0
},
"date": {
"cases": 8,
"mean_wer": 0.0367,
"doubtful_vowels": 6,
"vowels": 275,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 651.7
},
"order_id": {
"cases": 5,
"mean_wer": 0.2327,
"doubtful_vowels": 0,
"vowels": 133,
"unread_words": 0,
"digit_accuracy": 1.0,
"times_real_time": 839.2
},
"question": {
"cases": 3,
"mean_wer": 0.0278,
"doubtful_vowels": 0,
"vowels": 65,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 452.0
},
"arabic": {
"cases": 6,
"mean_wer": 0.0,
"doubtful_vowels": 0,
"vowels": 116,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 543.0
},
"code_switch": {
"cases": 4,
"mean_wer": 0.1131,
"doubtful_vowels": 0,
"vowels": 65,
"unread_words": 1,
"digit_accuracy": null,
"times_real_time": 485.0
}
},
"digit_accuracy": 1.0,
"judge2_vowels": 711,
"judge2_agreed": 703,
"judge2_agreement": 0.9887
},
"teacher": {
"cases": 43,
"mean_wer": 0.07,
"judge1_vowels": 1375,
"judge1_doubtful": 14,
"judge1_agreement": 0.9898,
"digits_expected": 67,
"digits_correct": 65,
"kinds": {
"short": {
"cases": 3,
"mean_wer": 0.0,
"doubtful_vowels": 1,
"vowels": 25,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 0.7
},
"medium": {
"cases": 3,
"mean_wer": 0.0256,
"doubtful_vowels": 2,
"vowels": 89,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 2.6
},
"long": {
"cases": 3,
"mean_wer": 0.0392,
"doubtful_vowels": 1,
"vowels": 306,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 1.3
},
"numbers": {
"cases": 3,
"mean_wer": 0.0,
"doubtful_vowels": 1,
"vowels": 132,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 0.3
},
"phone": {
"cases": 5,
"mean_wer": 0.0731,
"doubtful_vowels": 7,
"vowels": 169,
"unread_words": 0,
"digit_accuracy": 0.98,
"times_real_time": 0.3
},
"date": {
"cases": 8,
"mean_wer": 0.1557,
"doubtful_vowels": 1,
"vowels": 275,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 0.3
},
"order_id": {
"cases": 5,
"mean_wer": 0.1,
"doubtful_vowels": 1,
"vowels": 133,
"unread_words": 0,
"digit_accuracy": 0.95,
"times_real_time": 0.3
},
"question": {
"cases": 3,
"mean_wer": 0.0278,
"doubtful_vowels": 0,
"vowels": 65,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 0.2
},
"arabic": {
"cases": 6,
"mean_wer": 0.0278,
"doubtful_vowels": 0,
"vowels": 116,
"unread_words": 0,
"digit_accuracy": null,
"times_real_time": 0.2
},
"code_switch": {
"cases": 4,
"mean_wer": 0.1131,
"doubtful_vowels": 0,
"vowels": 65,
"unread_words": 1,
"digit_accuracy": null,
"times_real_time": 0.2
}
},
"digit_accuracy": 0.9701,
"judge2_vowels": 684,
"judge2_agreed": 673,
"judge2_agreement": 0.9839
},
"h3": {
"source": "work/release/runs/v1/inference.pt",
"voice": "M2",
"cases": 92,
"pass": 91,
"fail": 1,
"undecidable": 0,
"pass_share": 0.9891,
"failed_words": [
"ุนูู‚ู’ุฏูŒ"
],
"passed": 91,
"failed": 1,
"share": 0.9891
},
"h2": {
"source": "teacher",
"voice": "M2",
"cases": 92,
"pass": 90,
"fail": 2,
"undecidable": 0,
"pass_share": 0.9783,
"failed_words": [
"ุนูู‚ู’ุฏูŒ",
"ูŠูุนู’ู„ูŽู…ู"
],
"passed": 90,
"failed": 2,
"share": 0.9783
},
"corpus": {
"accepted": 6179,
"rejected": 1550,
"accepted_hours": 12.12,
"rejection_rate": 0.2005,
"classical_share": "80%",
"structured_share": "17%",
"validation": 123,
"categories": {
"classical": 4934,
"phone": 320,
"date": 294,
"order_id": 192,
"otp": 122,
"numbers": 80,
"prose": 52,
"iban": 35,
"talk": 34,
"clock": 26,
"question": 22,
"news": 21,
"names": 13,
"short": 10,
"grammar_drill": 6,
"support_apology": 6,
"long": 3,
"backchannel": 3,
"clause_flow": 2,
"poetry": 2,
"banking": 2
},
"rejection_reasons": {
"asr_mismatch": 1015,
"haraka_mismatch": 467,
"digit_mismatch": 67,
"duration": 1
},
"generation_seconds_final_attempts": 33580,
"mean_wer_accepted": 0.0274
},
"gates": [
{
"gate": "digits 100% on phone and order-id cases",
"passed": true,
"detail": "67/67"
},
{
"gate": "H3 minimal pairs >= 98%",
"passed": true,
"detail": "91/92"
},
{
"gate": "WER within 2 points of the teacher",
"passed": true,
"detail": "student 0.0532 vs teacher 0.07"
}
]
}