KapiCoder-Mongo-Small / evaluation /val_v2c_summary.json
esilva's picture
Replace with v4b weights (v0.3.0-preview)
2419a3b verified
Raw History Blame Contribute Delete
20.6 kB
{
"schema": "kapi-hf-eval-summary/1",
"kind": "val_v2c",
"label": "Internal evaluation on synthetic data; no blind human-written set. Aggregates only; no item text.",
"model": "esilva/KapiCoder-Mongo-Small",
"split": "val",
"n": 2000,
"seed": 20261008,
"contract": "editor-context-v1",
"dataset": {
"sample_ids_sha256": "28ba86be496839a321de0035d6a7e21827a5e8de4362151d80958ee8c798562f",
"sha256": "af3331481966f3ebfb05d36109b43014cd850f669f6f5d2a524f8da6e9609ec7",
"manifest_sha256": "2fef4f342d8a21eaf6d7e9eb1f223b3c8094d56c8dcd00e64f04bc20b20cd3cb",
"stratify_by": [
"cut_kind",
"domain"
],
"cut_kind": {
"multiline": 161,
"key": 319,
"value": 280,
"rest": 241,
"operator": 268,
"member": 201,
"stage": 251,
"partial": 200,
"stmt_start": 79
},
"domain": {
"commerce": 168,
"content": 161,
"energy": 171,
"finance": 175,
"gaming": 163,
"healthcare-synthetic": 173,
"hr": 168,
"iot": 168,
"logistics": 157,
"saas": 165,
"travel": 162,
"workspace": 169
}
},
"decoding": "greedy (argmax), padding do vocab mascarado",
"max_new_tokens": 96,
"context_tokens": 2048,
"load": "bf16",
"this_model": {
"headline": {
"n": 2000,
"exact_match": 0.399,
"normalized_match": 0.399,
"prefix_match": 0.4095,
"edit_similarity": 0.6413,
"chars_accepted": 7.8105,
"tab_chars": 6.312,
"accepted_prefix_tokens": 2.566,
"syntax_valid": 0.9905,
"mongo_valid": 0.9875,
"exec_valid": 0.5815,
"exec_valid_incl_empty": 0.5815,
"nonexistent_fields_rate": 0.034,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.991,
"mean_generated_tokens": 14.7685
},
"headline_ide32": {
"n": 2000,
"exact_match": 0.398,
"normalized_match": 0.398,
"prefix_match": 0.4095,
"edit_similarity": 0.6355,
"chars_accepted": 7.798,
"tab_chars": 6.2995,
"accepted_prefix_tokens": 2.5615,
"syntax_valid": 0.864,
"mongo_valid": 0.861,
"exec_valid": 0.5095,
"exec_valid_incl_empty": 0.5095,
"nonexistent_fields_rate": 0.023,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": null,
"mean_generated_tokens": null
},
"reference_ceiling": {
"syntax_valid": 1.0,
"mongo_valid": 1.0,
"exec_valid": 0.6255,
"nonexistent_fields_rate": 0.0
},
"exec_stage_distribution": {
"ok": 1163,
"api": 480,
"syntax": 258,
"fields": 92,
"exec": 3,
"type": 4
},
"by_cut_kind": {
"key": {
"n": 319,
"exact_match": 0.4357,
"normalized_match": 0.4357,
"prefix_match": 0.442,
"edit_similarity": 0.6301,
"chars_accepted": 6.1912,
"tab_chars": 5.6207,
"accepted_prefix_tokens": 1.9154,
"syntax_valid": 0.9906,
"mongo_valid": 0.9843,
"exec_valid": 0.5204,
"exec_valid_incl_empty": 0.5204,
"nonexistent_fields_rate": 0.0502,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9937,
"mean_generated_tokens": 8.4013
},
"member": {
"n": 201,
"exact_match": 0.7811,
"normalized_match": 0.7811,
"prefix_match": 0.7861,
"edit_similarity": 0.8511,
"chars_accepted": 8.3234,
"tab_chars": 8.0597,
"accepted_prefix_tokens": 1.7413,
"syntax_valid": 1.0,
"mongo_valid": 0.995,
"exec_valid": 0.6368,
"exec_valid_incl_empty": 0.6368,
"nonexistent_fields_rate": 0.005,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 1.0,
"mean_generated_tokens": 4.9701
},
"multiline": {
"n": 161,
"exact_match": 0.0,
"normalized_match": 0.0,
"prefix_match": 0.0,
"edit_similarity": 0.3836,
"chars_accepted": 9.528,
"tab_chars": 5.8944,
"accepted_prefix_tokens": 2.882,
"syntax_valid": 0.9814,
"mongo_valid": 0.9814,
"exec_valid": 0.5839,
"exec_valid_incl_empty": 0.5839,
"nonexistent_fields_rate": 0.1118,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9814,
"mean_generated_tokens": 41.6584
},
"operator": {
"n": 268,
"exact_match": 0.5896,
"normalized_match": 0.5896,
"prefix_match": 0.6045,
"edit_similarity": 0.7908,
"chars_accepted": 7.8657,
"tab_chars": 6.5261,
"accepted_prefix_tokens": 3.1082,
"syntax_valid": 0.9925,
"mongo_valid": 0.9888,
"exec_valid": 0.6157,
"exec_valid_incl_empty": 0.6157,
"nonexistent_fields_rate": 0.0224,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9925,
"mean_generated_tokens": 9.597
},
"partial": {
"n": 200,
"exact_match": 0.73,
"normalized_match": 0.73,
"prefix_match": 0.735,
"edit_similarity": 0.8495,
"chars_accepted": 6.075,
"tab_chars": 5.375,
"accepted_prefix_tokens": 2.3,
"syntax_valid": 0.995,
"mongo_valid": 0.99,
"exec_valid": 0.585,
"exec_valid_incl_empty": 0.585,
"nonexistent_fields_rate": 0.035,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 1.0,
"mean_generated_tokens": 7.175
},
"rest": {
"n": 241,
"exact_match": 0.3402,
"normalized_match": 0.3402,
"prefix_match": 0.3444,
"edit_similarity": 0.6247,
"chars_accepted": 9.2946,
"tab_chars": 7.3237,
"accepted_prefix_tokens": 2.9087,
"syntax_valid": 0.9917,
"mongo_valid": 0.9876,
"exec_valid": 0.5685,
"exec_valid_incl_empty": 0.5685,
"nonexistent_fields_rate": 0.0166,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9959,
"mean_generated_tokens": 14.7676
},
"stage": {
"n": 251,
"exact_match": 0.0159,
"normalized_match": 0.0159,
"prefix_match": 0.0159,
"edit_similarity": 0.4148,
"chars_accepted": 11.004,
"tab_chars": 7.753,
"accepted_prefix_tokens": 4.0637,
"syntax_valid": 0.9761,
"mongo_valid": 0.9761,
"exec_valid": 0.5538,
"exec_valid_incl_empty": 0.5538,
"nonexistent_fields_rate": 0.0438,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9641,
"mean_generated_tokens": 30.9323
},
"stmt_start": {
"n": 79,
"exact_match": 0.3165,
"normalized_match": 0.3165,
"prefix_match": 0.3165,
"edit_similarity": 0.6021,
"chars_accepted": 16.1772,
"tab_chars": 14.5696,
"accepted_prefix_tokens": 3.6709,
"syntax_valid": 1.0,
"mongo_valid": 1.0,
"exec_valid": 0.6456,
"exec_valid_incl_empty": 0.6456,
"nonexistent_fields_rate": 0.0633,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 1.0,
"mean_generated_tokens": 16.4304
},
"value": {
"n": 280,
"exact_match": 0.3107,
"normalized_match": 0.3107,
"prefix_match": 0.3536,
"edit_similarity": 0.5887,
"chars_accepted": 2.9857,
"tab_chars": 2.0571,
"accepted_prefix_tokens": 1.4393,
"syntax_valid": 0.9929,
"mongo_valid": 0.9929,
"exec_valid": 0.5929,
"exec_valid_incl_empty": 0.5929,
"nonexistent_fields_rate": 0.0,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9964,
"mean_generated_tokens": 9.0107
}
},
"by_domain": {
"commerce": {
"n": 168,
"exact_match": 0.381,
"normalized_match": 0.381,
"prefix_match": 0.3929,
"edit_similarity": 0.6256,
"chars_accepted": 7.1548,
"tab_chars": 5.6012,
"accepted_prefix_tokens": 2.256,
"syntax_valid": 1.0,
"mongo_valid": 0.9881,
"exec_valid": 0.6607,
"exec_valid_incl_empty": 0.6607,
"nonexistent_fields_rate": 0.0119,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 1.0,
"mean_generated_tokens": 14.875
},
"content": {
"n": 161,
"exact_match": 0.3665,
"normalized_match": 0.3665,
"prefix_match": 0.3851,
"edit_similarity": 0.6143,
"chars_accepted": 6.9876,
"tab_chars": 5.6646,
"accepted_prefix_tokens": 2.2795,
"syntax_valid": 0.9938,
"mongo_valid": 0.9876,
"exec_valid": 0.5839,
"exec_valid_incl_empty": 0.5839,
"nonexistent_fields_rate": 0.0248,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9876,
"mean_generated_tokens": 13.5528
},
"energy": {
"n": 171,
"exact_match": 0.4152,
"normalized_match": 0.4152,
"prefix_match": 0.4327,
"edit_similarity": 0.654,
"chars_accepted": 8.0409,
"tab_chars": 6.883,
"accepted_prefix_tokens": 2.4386,
"syntax_valid": 0.9883,
"mongo_valid": 0.9883,
"exec_valid": 0.5848,
"exec_valid_incl_empty": 0.5848,
"nonexistent_fields_rate": 0.0526,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9825,
"mean_generated_tokens": 15.6959
},
"finance": {
"n": 175,
"exact_match": 0.3543,
"normalized_match": 0.3543,
"prefix_match": 0.36,
"edit_similarity": 0.604,
"chars_accepted": 8.2286,
"tab_chars": 6.8057,
"accepted_prefix_tokens": 2.9086,
"syntax_valid": 0.9829,
"mongo_valid": 0.9829,
"exec_valid": 0.5486,
"exec_valid_incl_empty": 0.5486,
"nonexistent_fields_rate": 0.0686,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9943,
"mean_generated_tokens": 16.8743
},
"gaming": {
"n": 163,
"exact_match": 0.3681,
"normalized_match": 0.3681,
"prefix_match": 0.3804,
"edit_similarity": 0.6173,
"chars_accepted": 7.4356,
"tab_chars": 6.0613,
"accepted_prefix_tokens": 2.454,
"syntax_valid": 0.9816,
"mongo_valid": 0.9693,
"exec_valid": 0.5828,
"exec_valid_incl_empty": 0.5828,
"nonexistent_fields_rate": 0.0368,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9816,
"mean_generated_tokens": 15.5706
},
"healthcare-synthetic": {
"n": 173,
"exact_match": 0.3988,
"normalized_match": 0.3988,
"prefix_match": 0.4104,
"edit_similarity": 0.6482,
"chars_accepted": 8.341,
"tab_chars": 6.3757,
"accepted_prefix_tokens": 2.815,
"syntax_valid": 0.9884,
"mongo_valid": 0.9884,
"exec_valid": 0.5723,
"exec_valid_incl_empty": 0.5723,
"nonexistent_fields_rate": 0.0116,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9827,
"mean_generated_tokens": 14.5087
},
"hr": {
"n": 168,
"exact_match": 0.4643,
"normalized_match": 0.4643,
"prefix_match": 0.4762,
"edit_similarity": 0.6813,
"chars_accepted": 8.2976,
"tab_chars": 7.0952,
"accepted_prefix_tokens": 2.6607,
"syntax_valid": 1.0,
"mongo_valid": 1.0,
"exec_valid": 0.5714,
"exec_valid_incl_empty": 0.5714,
"nonexistent_fields_rate": 0.0417,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 1.0,
"mean_generated_tokens": 11.6488
},
"iot": {
"n": 168,
"exact_match": 0.4167,
"normalized_match": 0.4167,
"prefix_match": 0.4226,
"edit_similarity": 0.6363,
"chars_accepted": 8.0714,
"tab_chars": 6.4048,
"accepted_prefix_tokens": 2.6964,
"syntax_valid": 1.0,
"mongo_valid": 1.0,
"exec_valid": 0.6369,
"exec_valid_incl_empty": 0.6369,
"nonexistent_fields_rate": 0.0476,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 1.0,
"mean_generated_tokens": 15.1905
},
"logistics": {
"n": 157,
"exact_match": 0.4076,
"normalized_match": 0.4076,
"prefix_match": 0.4204,
"edit_similarity": 0.6843,
"chars_accepted": 8.2739,
"tab_chars": 6.3503,
"accepted_prefix_tokens": 2.7134,
"syntax_valid": 1.0,
"mongo_valid": 0.9936,
"exec_valid": 0.5924,
"exec_valid_incl_empty": 0.5924,
"nonexistent_fields_rate": 0.0318,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 1.0,
"mean_generated_tokens": 13.3312
},
"saas": {
"n": 165,
"exact_match": 0.4788,
"normalized_match": 0.4788,
"prefix_match": 0.4848,
"edit_similarity": 0.6914,
"chars_accepted": 7.3152,
"tab_chars": 5.5939,
"accepted_prefix_tokens": 2.497,
"syntax_valid": 0.9758,
"mongo_valid": 0.9758,
"exec_valid": 0.4788,
"exec_valid_incl_empty": 0.4788,
"nonexistent_fields_rate": 0.0303,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9879,
"mean_generated_tokens": 13.4485
},
"travel": {
"n": 162,
"exact_match": 0.3889,
"normalized_match": 0.3889,
"prefix_match": 0.3951,
"edit_similarity": 0.6216,
"chars_accepted": 7.8272,
"tab_chars": 6.4383,
"accepted_prefix_tokens": 2.6173,
"syntax_valid": 1.0,
"mongo_valid": 1.0,
"exec_valid": 0.6296,
"exec_valid_incl_empty": 0.6296,
"nonexistent_fields_rate": 0.0247,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 1.0,
"mean_generated_tokens": 15.7037
},
"workspace": {
"n": 169,
"exact_match": 0.3491,
"normalized_match": 0.3491,
"prefix_match": 0.355,
"edit_similarity": 0.6204,
"chars_accepted": 7.6923,
"tab_chars": 6.3964,
"accepted_prefix_tokens": 2.432,
"syntax_valid": 0.9763,
"mongo_valid": 0.9763,
"exec_valid": 0.5385,
"exec_valid_incl_empty": 0.5385,
"nonexistent_fields_rate": 0.0237,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9763,
"mean_generated_tokens": 16.6036
}
},
"by_context_kind": {
"full": {
"n": 1647,
"exact_match": 0.408,
"normalized_match": 0.408,
"prefix_match": 0.4189,
"edit_similarity": 0.6525,
"chars_accepted": 8.0291,
"tab_chars": 6.5209,
"accepted_prefix_tokens": 2.6436,
"syntax_valid": 0.9897,
"mongo_valid": 0.9879,
"exec_valid": 0.6072,
"exec_valid_incl_empty": 0.6072,
"nonexistent_fields_rate": 0.0182,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9909,
"mean_generated_tokens": 14.8336
},
"irrelevant": {
"n": 198,
"exact_match": 0.3636,
"normalized_match": 0.3636,
"prefix_match": 0.3687,
"edit_similarity": 0.584,
"chars_accepted": 7.1111,
"tab_chars": 5.8232,
"accepted_prefix_tokens": 2.2222,
"syntax_valid": 0.9899,
"mongo_valid": 0.9798,
"exec_valid": 0.4747,
"exec_valid_incl_empty": 0.4747,
"nonexistent_fields_rate": 0.096,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9899,
"mean_generated_tokens": 14.5354
},
"none": {
"n": 155,
"exact_match": 0.3484,
"normalized_match": 0.3484,
"prefix_match": 0.3613,
"edit_similarity": 0.5964,
"chars_accepted": 6.3806,
"tab_chars": 4.7161,
"accepted_prefix_tokens": 2.1806,
"syntax_valid": 1.0,
"mongo_valid": 0.9935,
"exec_valid": 0.4452,
"exec_valid_incl_empty": 0.4452,
"nonexistent_fields_rate": 0.1226,
"empty_rate": 0.0,
"eos_or_suffix_stop_rate": 0.9935,
"mean_generated_tokens": 14.3742
}
}
},
"metrics_by_model": {
"this_model_v4b": {
"syntax_valid": 0.9905,
"mongodb_valid": 0.9875,
"nonexistent_fields_rate": 3.4,
"conversational_suggestion_rate": 0.0,
"fim_acc": 0.399,
"exact_match": 0.399,
"apt_mean": 2.566,
"exec_valid_autocomplete": 0.5815,
"empty_rate": 0.0,
"reference_ceiling_exec_valid": 0.6255,
"exec_valid_vs_ceiling": 0.9297,
"nonexistent_fields_full_rate_pct": 1.82,
"n_context_full": 1647,
"exec_valid_full": 0.6072,
"nonexistent_fields_irrelevant_rate_pct": 9.6,
"n_context_irrelevant": 198,
"exec_valid_irrelevant": 0.4747,
"nonexistent_fields_none_rate_pct": 12.26,
"n_context_none": 155,
"exec_valid_none": 0.4452,
"info_fim_acc_gain_vs_zero_shot_base_pp": 25.7,
"info_apt_delta_vs_zero_shot_base": 1.127,
"info_exec_valid_delta_vs_zero_shot_base_pp": 26.55,
"generic_js_fim_acc": 0.626,
"generic_js_fim_delta_pp_vs_zero_shot_base": 22.0,
"write_or_destructive_executions": 0,
"train_eval_leak_count": 0,
"decontamination_approved": true,
"auto_report_attached": true
},
"previous_release_v4_v0.2.0_preview": {
"syntax_valid": 0.99,
"mongodb_valid": 0.987,
"nonexistent_fields_rate": 3.75,
"conversational_suggestion_rate": 0.0,
"fim_acc": 0.4065,
"exact_match": 0.4065,
"apt_mean": 2.6265,
"exec_valid_autocomplete": 0.5745,
"empty_rate": 0.0,
"reference_ceiling_exec_valid": 0.6255,
"exec_valid_vs_ceiling": 0.9185,
"nonexistent_fields_full_rate_pct": 2.0,
"n_context_full": 1647,
"exec_valid_full": 0.5981,
"nonexistent_fields_irrelevant_rate_pct": 12.63,
"n_context_irrelevant": 198,
"exec_valid_irrelevant": 0.4596,
"nonexistent_fields_none_rate_pct": 10.97,
"n_context_none": 155,
"exec_valid_none": 0.471,
"info_fim_acc_gain_vs_zero_shot_base_pp": 26.45,
"info_apt_delta_vs_zero_shot_base": 1.1875,
"info_exec_valid_delta_vs_zero_shot_base_pp": 25.85,
"generic_js_fim_acc": 0.628,
"generic_js_fim_delta_pp_vs_zero_shot_base": 22.2,
"write_or_destructive_executions": 0,
"train_eval_leak_count": 0,
"decontamination_approved": true,
"auto_report_attached": true
},
"zero_shot_base": {
"syntax_valid": 0.667,
"mongodb_valid": 0.595,
"nonexistent_fields_rate": 5.1,
"conversational_suggestion_rate": 0.0,
"fim_acc": 0.142,
"exact_match": 0.142,
"apt_mean": 1.439,
"exec_valid_autocomplete": 0.316,
"empty_rate": 0.0095,
"reference_ceiling_exec_valid": 0.6255,
"exec_valid_vs_ceiling": 0.5052,
"nonexistent_fields_full_rate_pct": 3.76,
"n_context_full": 1647,
"exec_valid_full": 0.3364,
"nonexistent_fields_irrelevant_rate_pct": 10.61,
"n_context_irrelevant": 198,
"exec_valid_irrelevant": 0.2172,
"nonexistent_fields_none_rate_pct": 12.26,
"n_context_none": 155,
"exec_valid_none": 0.2258,
"generic_js_fim_acc": 0.406,
"write_or_destructive_executions": 0,
"train_eval_leak_count": 0,
"decontamination_approved": true,
"auto_report_attached": true
}
},
"metric_definitions": {
"exact_match": "pred.strip() == middle.strip()",
"normalized_match": "igualdade com espacos normalizados (FIM accuracy do bake-off)",
"chars_accepted": "tamanho do prefixo comum (caracteres) entre predicao e middle",
"accepted_prefix_tokens": "tokens iniciais da referencia cobertos pelo prefixo comum (APT)",
"syntax_valid": "node vm.Script sobre prefix+pred+suffix",
"mongo_valid": "mongo_validate (AST, estatico) sobre o documento reconstruido",
"exec_valid": "chat_validate.validate_text stage=ok (sintaxe, API do Console, campos, tipos, execucao somente-leitura em kapi_fx_*) E predicao nao vazia; exec_valid_incl_empty conta vazio como valido",
"nonexistent_fields_rate": "stage=fields com 'campo/estrutura invalido' (campo ausente do schema) sobre todos os exemplos",
"reference_ceiling": "mesmas checagens aplicadas ao middle de referencia (teto/calibracao do validador)"
}
}