{ "schema": "kapi-hf-eval-summary/1", "kind": "val_v2c", "label": "Internal evaluation on synthetic data; no blind human-written set. Aggregates only; no item text.", "model": "esilva/KapiCoder-Mongo-Small", "split": "val", "n": 2000, "seed": 20261008, "contract": "editor-context-v1", "dataset": { "sample_ids_sha256": "28ba86be496839a321de0035d6a7e21827a5e8de4362151d80958ee8c798562f", "sha256": "af3331481966f3ebfb05d36109b43014cd850f669f6f5d2a524f8da6e9609ec7", "manifest_sha256": "2fef4f342d8a21eaf6d7e9eb1f223b3c8094d56c8dcd00e64f04bc20b20cd3cb", "stratify_by": [ "cut_kind", "domain" ], "cut_kind": { "multiline": 161, "key": 319, "value": 280, "rest": 241, "operator": 268, "member": 201, "stage": 251, "partial": 200, "stmt_start": 79 }, "domain": { "commerce": 168, "content": 161, "energy": 171, "finance": 175, "gaming": 163, "healthcare-synthetic": 173, "hr": 168, "iot": 168, "logistics": 157, "saas": 165, "travel": 162, "workspace": 169 } }, "decoding": "greedy (argmax), padding do vocab mascarado", "max_new_tokens": 96, "context_tokens": 2048, "load": "bf16", "this_model": { "headline": { "n": 2000, "exact_match": 0.399, "normalized_match": 0.399, "prefix_match": 0.4095, "edit_similarity": 0.6413, "chars_accepted": 7.8105, "tab_chars": 6.312, "accepted_prefix_tokens": 2.566, "syntax_valid": 0.9905, "mongo_valid": 0.9875, "exec_valid": 0.5815, "exec_valid_incl_empty": 0.5815, "nonexistent_fields_rate": 0.034, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.991, "mean_generated_tokens": 14.7685 }, "headline_ide32": { "n": 2000, "exact_match": 0.398, "normalized_match": 0.398, "prefix_match": 0.4095, "edit_similarity": 0.6355, "chars_accepted": 7.798, "tab_chars": 6.2995, "accepted_prefix_tokens": 2.5615, "syntax_valid": 0.864, "mongo_valid": 0.861, "exec_valid": 0.5095, "exec_valid_incl_empty": 0.5095, "nonexistent_fields_rate": 0.023, "empty_rate": 0.0, "eos_or_suffix_stop_rate": null, "mean_generated_tokens": null }, "reference_ceiling": { "syntax_valid": 1.0, "mongo_valid": 1.0, "exec_valid": 0.6255, "nonexistent_fields_rate": 0.0 }, "exec_stage_distribution": { "ok": 1163, "api": 480, "syntax": 258, "fields": 92, "exec": 3, "type": 4 }, "by_cut_kind": { "key": { "n": 319, "exact_match": 0.4357, "normalized_match": 0.4357, "prefix_match": 0.442, "edit_similarity": 0.6301, "chars_accepted": 6.1912, "tab_chars": 5.6207, "accepted_prefix_tokens": 1.9154, "syntax_valid": 0.9906, "mongo_valid": 0.9843, "exec_valid": 0.5204, "exec_valid_incl_empty": 0.5204, "nonexistent_fields_rate": 0.0502, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9937, "mean_generated_tokens": 8.4013 }, "member": { "n": 201, "exact_match": 0.7811, "normalized_match": 0.7811, "prefix_match": 0.7861, "edit_similarity": 0.8511, "chars_accepted": 8.3234, "tab_chars": 8.0597, "accepted_prefix_tokens": 1.7413, "syntax_valid": 1.0, "mongo_valid": 0.995, "exec_valid": 0.6368, "exec_valid_incl_empty": 0.6368, "nonexistent_fields_rate": 0.005, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 1.0, "mean_generated_tokens": 4.9701 }, "multiline": { "n": 161, "exact_match": 0.0, "normalized_match": 0.0, "prefix_match": 0.0, "edit_similarity": 0.3836, "chars_accepted": 9.528, "tab_chars": 5.8944, "accepted_prefix_tokens": 2.882, "syntax_valid": 0.9814, "mongo_valid": 0.9814, "exec_valid": 0.5839, "exec_valid_incl_empty": 0.5839, "nonexistent_fields_rate": 0.1118, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9814, "mean_generated_tokens": 41.6584 }, "operator": { "n": 268, "exact_match": 0.5896, "normalized_match": 0.5896, "prefix_match": 0.6045, "edit_similarity": 0.7908, "chars_accepted": 7.8657, "tab_chars": 6.5261, "accepted_prefix_tokens": 3.1082, "syntax_valid": 0.9925, "mongo_valid": 0.9888, "exec_valid": 0.6157, "exec_valid_incl_empty": 0.6157, "nonexistent_fields_rate": 0.0224, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9925, "mean_generated_tokens": 9.597 }, "partial": { "n": 200, "exact_match": 0.73, "normalized_match": 0.73, "prefix_match": 0.735, "edit_similarity": 0.8495, "chars_accepted": 6.075, "tab_chars": 5.375, "accepted_prefix_tokens": 2.3, "syntax_valid": 0.995, "mongo_valid": 0.99, "exec_valid": 0.585, "exec_valid_incl_empty": 0.585, "nonexistent_fields_rate": 0.035, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 1.0, "mean_generated_tokens": 7.175 }, "rest": { "n": 241, "exact_match": 0.3402, "normalized_match": 0.3402, "prefix_match": 0.3444, "edit_similarity": 0.6247, "chars_accepted": 9.2946, "tab_chars": 7.3237, "accepted_prefix_tokens": 2.9087, "syntax_valid": 0.9917, "mongo_valid": 0.9876, "exec_valid": 0.5685, "exec_valid_incl_empty": 0.5685, "nonexistent_fields_rate": 0.0166, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9959, "mean_generated_tokens": 14.7676 }, "stage": { "n": 251, "exact_match": 0.0159, "normalized_match": 0.0159, "prefix_match": 0.0159, "edit_similarity": 0.4148, "chars_accepted": 11.004, "tab_chars": 7.753, "accepted_prefix_tokens": 4.0637, "syntax_valid": 0.9761, "mongo_valid": 0.9761, "exec_valid": 0.5538, "exec_valid_incl_empty": 0.5538, "nonexistent_fields_rate": 0.0438, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9641, "mean_generated_tokens": 30.9323 }, "stmt_start": { "n": 79, "exact_match": 0.3165, "normalized_match": 0.3165, "prefix_match": 0.3165, "edit_similarity": 0.6021, "chars_accepted": 16.1772, "tab_chars": 14.5696, "accepted_prefix_tokens": 3.6709, "syntax_valid": 1.0, "mongo_valid": 1.0, "exec_valid": 0.6456, "exec_valid_incl_empty": 0.6456, "nonexistent_fields_rate": 0.0633, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 1.0, "mean_generated_tokens": 16.4304 }, "value": { "n": 280, "exact_match": 0.3107, "normalized_match": 0.3107, "prefix_match": 0.3536, "edit_similarity": 0.5887, "chars_accepted": 2.9857, "tab_chars": 2.0571, "accepted_prefix_tokens": 1.4393, "syntax_valid": 0.9929, "mongo_valid": 0.9929, "exec_valid": 0.5929, "exec_valid_incl_empty": 0.5929, "nonexistent_fields_rate": 0.0, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9964, "mean_generated_tokens": 9.0107 } }, "by_domain": { "commerce": { "n": 168, "exact_match": 0.381, "normalized_match": 0.381, "prefix_match": 0.3929, "edit_similarity": 0.6256, "chars_accepted": 7.1548, "tab_chars": 5.6012, "accepted_prefix_tokens": 2.256, "syntax_valid": 1.0, "mongo_valid": 0.9881, "exec_valid": 0.6607, "exec_valid_incl_empty": 0.6607, "nonexistent_fields_rate": 0.0119, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 1.0, "mean_generated_tokens": 14.875 }, "content": { "n": 161, "exact_match": 0.3665, "normalized_match": 0.3665, "prefix_match": 0.3851, "edit_similarity": 0.6143, "chars_accepted": 6.9876, "tab_chars": 5.6646, "accepted_prefix_tokens": 2.2795, "syntax_valid": 0.9938, "mongo_valid": 0.9876, "exec_valid": 0.5839, "exec_valid_incl_empty": 0.5839, "nonexistent_fields_rate": 0.0248, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9876, "mean_generated_tokens": 13.5528 }, "energy": { "n": 171, "exact_match": 0.4152, "normalized_match": 0.4152, "prefix_match": 0.4327, "edit_similarity": 0.654, "chars_accepted": 8.0409, "tab_chars": 6.883, "accepted_prefix_tokens": 2.4386, "syntax_valid": 0.9883, "mongo_valid": 0.9883, "exec_valid": 0.5848, "exec_valid_incl_empty": 0.5848, "nonexistent_fields_rate": 0.0526, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9825, "mean_generated_tokens": 15.6959 }, "finance": { "n": 175, "exact_match": 0.3543, "normalized_match": 0.3543, "prefix_match": 0.36, "edit_similarity": 0.604, "chars_accepted": 8.2286, "tab_chars": 6.8057, "accepted_prefix_tokens": 2.9086, "syntax_valid": 0.9829, "mongo_valid": 0.9829, "exec_valid": 0.5486, "exec_valid_incl_empty": 0.5486, "nonexistent_fields_rate": 0.0686, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9943, "mean_generated_tokens": 16.8743 }, "gaming": { "n": 163, "exact_match": 0.3681, "normalized_match": 0.3681, "prefix_match": 0.3804, "edit_similarity": 0.6173, "chars_accepted": 7.4356, "tab_chars": 6.0613, "accepted_prefix_tokens": 2.454, "syntax_valid": 0.9816, "mongo_valid": 0.9693, "exec_valid": 0.5828, "exec_valid_incl_empty": 0.5828, "nonexistent_fields_rate": 0.0368, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9816, "mean_generated_tokens": 15.5706 }, "healthcare-synthetic": { "n": 173, "exact_match": 0.3988, "normalized_match": 0.3988, "prefix_match": 0.4104, "edit_similarity": 0.6482, "chars_accepted": 8.341, "tab_chars": 6.3757, "accepted_prefix_tokens": 2.815, "syntax_valid": 0.9884, "mongo_valid": 0.9884, "exec_valid": 0.5723, "exec_valid_incl_empty": 0.5723, "nonexistent_fields_rate": 0.0116, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9827, "mean_generated_tokens": 14.5087 }, "hr": { "n": 168, "exact_match": 0.4643, "normalized_match": 0.4643, "prefix_match": 0.4762, "edit_similarity": 0.6813, "chars_accepted": 8.2976, "tab_chars": 7.0952, "accepted_prefix_tokens": 2.6607, "syntax_valid": 1.0, "mongo_valid": 1.0, "exec_valid": 0.5714, "exec_valid_incl_empty": 0.5714, "nonexistent_fields_rate": 0.0417, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 1.0, "mean_generated_tokens": 11.6488 }, "iot": { "n": 168, "exact_match": 0.4167, "normalized_match": 0.4167, "prefix_match": 0.4226, "edit_similarity": 0.6363, "chars_accepted": 8.0714, "tab_chars": 6.4048, "accepted_prefix_tokens": 2.6964, "syntax_valid": 1.0, "mongo_valid": 1.0, "exec_valid": 0.6369, "exec_valid_incl_empty": 0.6369, "nonexistent_fields_rate": 0.0476, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 1.0, "mean_generated_tokens": 15.1905 }, "logistics": { "n": 157, "exact_match": 0.4076, "normalized_match": 0.4076, "prefix_match": 0.4204, "edit_similarity": 0.6843, "chars_accepted": 8.2739, "tab_chars": 6.3503, "accepted_prefix_tokens": 2.7134, "syntax_valid": 1.0, "mongo_valid": 0.9936, "exec_valid": 0.5924, "exec_valid_incl_empty": 0.5924, "nonexistent_fields_rate": 0.0318, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 1.0, "mean_generated_tokens": 13.3312 }, "saas": { "n": 165, "exact_match": 0.4788, "normalized_match": 0.4788, "prefix_match": 0.4848, "edit_similarity": 0.6914, "chars_accepted": 7.3152, "tab_chars": 5.5939, "accepted_prefix_tokens": 2.497, "syntax_valid": 0.9758, "mongo_valid": 0.9758, "exec_valid": 0.4788, "exec_valid_incl_empty": 0.4788, "nonexistent_fields_rate": 0.0303, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9879, "mean_generated_tokens": 13.4485 }, "travel": { "n": 162, "exact_match": 0.3889, "normalized_match": 0.3889, "prefix_match": 0.3951, "edit_similarity": 0.6216, "chars_accepted": 7.8272, "tab_chars": 6.4383, "accepted_prefix_tokens": 2.6173, "syntax_valid": 1.0, "mongo_valid": 1.0, "exec_valid": 0.6296, "exec_valid_incl_empty": 0.6296, "nonexistent_fields_rate": 0.0247, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 1.0, "mean_generated_tokens": 15.7037 }, "workspace": { "n": 169, "exact_match": 0.3491, "normalized_match": 0.3491, "prefix_match": 0.355, "edit_similarity": 0.6204, "chars_accepted": 7.6923, "tab_chars": 6.3964, "accepted_prefix_tokens": 2.432, "syntax_valid": 0.9763, "mongo_valid": 0.9763, "exec_valid": 0.5385, "exec_valid_incl_empty": 0.5385, "nonexistent_fields_rate": 0.0237, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9763, "mean_generated_tokens": 16.6036 } }, "by_context_kind": { "full": { "n": 1647, "exact_match": 0.408, "normalized_match": 0.408, "prefix_match": 0.4189, "edit_similarity": 0.6525, "chars_accepted": 8.0291, "tab_chars": 6.5209, "accepted_prefix_tokens": 2.6436, "syntax_valid": 0.9897, "mongo_valid": 0.9879, "exec_valid": 0.6072, "exec_valid_incl_empty": 0.6072, "nonexistent_fields_rate": 0.0182, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9909, "mean_generated_tokens": 14.8336 }, "irrelevant": { "n": 198, "exact_match": 0.3636, "normalized_match": 0.3636, "prefix_match": 0.3687, "edit_similarity": 0.584, "chars_accepted": 7.1111, "tab_chars": 5.8232, "accepted_prefix_tokens": 2.2222, "syntax_valid": 0.9899, "mongo_valid": 0.9798, "exec_valid": 0.4747, "exec_valid_incl_empty": 0.4747, "nonexistent_fields_rate": 0.096, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9899, "mean_generated_tokens": 14.5354 }, "none": { "n": 155, "exact_match": 0.3484, "normalized_match": 0.3484, "prefix_match": 0.3613, "edit_similarity": 0.5964, "chars_accepted": 6.3806, "tab_chars": 4.7161, "accepted_prefix_tokens": 2.1806, "syntax_valid": 1.0, "mongo_valid": 0.9935, "exec_valid": 0.4452, "exec_valid_incl_empty": 0.4452, "nonexistent_fields_rate": 0.1226, "empty_rate": 0.0, "eos_or_suffix_stop_rate": 0.9935, "mean_generated_tokens": 14.3742 } } }, "metrics_by_model": { "this_model_v4b": { "syntax_valid": 0.9905, "mongodb_valid": 0.9875, "nonexistent_fields_rate": 3.4, "conversational_suggestion_rate": 0.0, "fim_acc": 0.399, "exact_match": 0.399, "apt_mean": 2.566, "exec_valid_autocomplete": 0.5815, "empty_rate": 0.0, "reference_ceiling_exec_valid": 0.6255, "exec_valid_vs_ceiling": 0.9297, "nonexistent_fields_full_rate_pct": 1.82, "n_context_full": 1647, "exec_valid_full": 0.6072, "nonexistent_fields_irrelevant_rate_pct": 9.6, "n_context_irrelevant": 198, "exec_valid_irrelevant": 0.4747, "nonexistent_fields_none_rate_pct": 12.26, "n_context_none": 155, "exec_valid_none": 0.4452, "info_fim_acc_gain_vs_zero_shot_base_pp": 25.7, "info_apt_delta_vs_zero_shot_base": 1.127, "info_exec_valid_delta_vs_zero_shot_base_pp": 26.55, "generic_js_fim_acc": 0.626, "generic_js_fim_delta_pp_vs_zero_shot_base": 22.0, "write_or_destructive_executions": 0, "train_eval_leak_count": 0, "decontamination_approved": true, "auto_report_attached": true }, "previous_release_v4_v0.2.0_preview": { "syntax_valid": 0.99, "mongodb_valid": 0.987, "nonexistent_fields_rate": 3.75, "conversational_suggestion_rate": 0.0, "fim_acc": 0.4065, "exact_match": 0.4065, "apt_mean": 2.6265, "exec_valid_autocomplete": 0.5745, "empty_rate": 0.0, "reference_ceiling_exec_valid": 0.6255, "exec_valid_vs_ceiling": 0.9185, "nonexistent_fields_full_rate_pct": 2.0, "n_context_full": 1647, "exec_valid_full": 0.5981, "nonexistent_fields_irrelevant_rate_pct": 12.63, "n_context_irrelevant": 198, "exec_valid_irrelevant": 0.4596, "nonexistent_fields_none_rate_pct": 10.97, "n_context_none": 155, "exec_valid_none": 0.471, "info_fim_acc_gain_vs_zero_shot_base_pp": 26.45, "info_apt_delta_vs_zero_shot_base": 1.1875, "info_exec_valid_delta_vs_zero_shot_base_pp": 25.85, "generic_js_fim_acc": 0.628, "generic_js_fim_delta_pp_vs_zero_shot_base": 22.2, "write_or_destructive_executions": 0, "train_eval_leak_count": 0, "decontamination_approved": true, "auto_report_attached": true }, "zero_shot_base": { "syntax_valid": 0.667, "mongodb_valid": 0.595, "nonexistent_fields_rate": 5.1, "conversational_suggestion_rate": 0.0, "fim_acc": 0.142, "exact_match": 0.142, "apt_mean": 1.439, "exec_valid_autocomplete": 0.316, "empty_rate": 0.0095, "reference_ceiling_exec_valid": 0.6255, "exec_valid_vs_ceiling": 0.5052, "nonexistent_fields_full_rate_pct": 3.76, "n_context_full": 1647, "exec_valid_full": 0.3364, "nonexistent_fields_irrelevant_rate_pct": 10.61, "n_context_irrelevant": 198, "exec_valid_irrelevant": 0.2172, "nonexistent_fields_none_rate_pct": 12.26, "n_context_none": 155, "exec_valid_none": 0.2258, "generic_js_fim_acc": 0.406, "write_or_destructive_executions": 0, "train_eval_leak_count": 0, "decontamination_approved": true, "auto_report_attached": true } }, "metric_definitions": { "exact_match": "pred.strip() == middle.strip()", "normalized_match": "igualdade com espacos normalizados (FIM accuracy do bake-off)", "chars_accepted": "tamanho do prefixo comum (caracteres) entre predicao e middle", "accepted_prefix_tokens": "tokens iniciais da referencia cobertos pelo prefixo comum (APT)", "syntax_valid": "node vm.Script sobre prefix+pred+suffix", "mongo_valid": "mongo_validate (AST, estatico) sobre o documento reconstruido", "exec_valid": "chat_validate.validate_text stage=ok (sintaxe, API do Console, campos, tipos, execucao somente-leitura em kapi_fx_*) E predicao nao vazia; exec_valid_incl_empty conta vazio como valido", "nonexistent_fields_rate": "stage=fields com 'campo/estrutura invalido' (campo ausente do schema) sobre todos os exemplos", "reference_ceiling": "mesmas checagens aplicadas ao middle de referencia (teto/calibracao do validador)" } }