KapiCoder-Mongo-Small / evaluation /cursor_in_call_summary.json
esilva's picture
Replace with v4b weights (v0.3.0-preview)
2419a3b verified
Raw History Blame Contribute Delete
13.1 kB
{
"schema": "kapi-hf-eval-summary/1",
"kind": "cursor_in_call_v1",
"label": "Internal evaluation on synthetic data; no blind human-written set. Aggregates only; no item text.",
"model": "esilva/KapiCoder-Mongo-Small",
"suite_sha256_prefix": "e7b40a166c74",
"decoding": "greedy",
"limit_tokens": 64,
"runs": {
"t1_v3_ctxv2": {
"context_tokens": 2048,
"items": 1200,
"postprocess": "v2-close-inner",
"overall_at_64": {
"n": 1200,
"match": 0.065,
"suffix_consistent": 0.8783,
"balanced": 0.9383,
"op_valid": 0.8567,
"op_valid_n": 1200,
"field_grounded": 0.8605,
"field_grounded_n": 803,
"empty": 0.0
},
"by_cut_kind_at_64": {
"after_colon": {
"n": 150,
"match": 0.1667,
"suffix_consistent": 0.9933,
"balanced": 0.9933,
"op_valid": 0.9733,
"op_valid_n": 150,
"field_grounded": 0.675,
"field_grounded_n": 40,
"empty": 0.0
},
"after_comma": {
"n": 150,
"match": 0.0,
"suffix_consistent": 0.9933,
"balanced": 1.0,
"op_valid": 0.9867,
"op_valid_n": 150,
"field_grounded": 0.9267,
"field_grounded_n": 150,
"empty": 0.0
},
"after_dollar": {
"n": 150,
"match": 0.0,
"suffix_consistent": 0.8133,
"balanced": 0.9333,
"op_valid": 0.8067,
"op_valid_n": 150,
"field_grounded": 0.8571,
"field_grounded_n": 63,
"empty": 0.0
},
"after_open": {
"n": 150,
"match": 0.0467,
"suffix_consistent": 0.9933,
"balanced": 0.9933,
"op_valid": 0.98,
"op_valid_n": 150,
"field_grounded": 0.922,
"field_grounded_n": 141,
"empty": 0.0
},
"dotted_path": {
"n": 150,
"match": 0.0067,
"suffix_consistent": 0.9933,
"balanced": 0.9933,
"op_valid": 0.9867,
"op_valid_n": 150,
"field_grounded": 0.88,
"field_grounded_n": 150,
"empty": 0.0
},
"in_string": {
"n": 150,
"match": 0.0733,
"suffix_consistent": 0.4133,
"balanced": 0.66,
"op_valid": 0.4067,
"op_valid_n": 150,
"field_grounded": 0.8444,
"field_grounded_n": 45,
"empty": 0.0
},
"partial_ident": {
"n": 150,
"match": 0.1333,
"suffix_consistent": 0.9933,
"balanced": 0.9933,
"op_valid": 0.9267,
"op_valid_n": 150,
"field_grounded": 0.8067,
"field_grounded_n": 150,
"empty": 0.0
},
"partial_op": {
"n": 150,
"match": 0.0933,
"suffix_consistent": 0.8333,
"balanced": 0.94,
"op_valid": 0.7867,
"op_valid_n": 150,
"field_grounded": 0.7812,
"field_grounded_n": 64,
"empty": 0.0
}
},
"by_context_kind_at_64": {
"full": {
"n": 736,
"match": 0.0788,
"suffix_consistent": 0.8696,
"balanced": 0.928,
"op_valid": 0.8465,
"op_valid_n": 736,
"field_grounded": 0.9614,
"field_grounded_n": 492,
"empty": 0.0
},
"irrelevant": {
"n": 244,
"match": 0.0451,
"suffix_consistent": 0.873,
"balanced": 0.9508,
"op_valid": 0.8402,
"op_valid_n": 244,
"field_grounded": 0.8373,
"field_grounded_n": 166,
"empty": 0.0
},
"none": {
"n": 220,
"match": 0.0409,
"suffix_consistent": 0.9136,
"balanced": 0.9591,
"op_valid": 0.9091,
"op_valid_n": 220,
"field_grounded": 0.5448,
"field_grounded_n": 145,
"empty": 0.0
}
},
"by_target_distance_at_64": {}
},
"t1_v4b_ctxv2": {
"context_tokens": 2048,
"items": 1200,
"postprocess": "v2-close-inner",
"overall_at_64": {
"n": 1200,
"match": 0.0792,
"suffix_consistent": 0.9492,
"balanced": 0.9775,
"op_valid": 0.9483,
"op_valid_n": 1200,
"field_grounded": 0.9051,
"field_grounded_n": 780,
"empty": 0.0
},
"by_cut_kind_at_64": {
"after_colon": {
"n": 150,
"match": 0.1533,
"suffix_consistent": 0.9867,
"balanced": 0.9933,
"op_valid": 0.9867,
"op_valid_n": 150,
"field_grounded": 0.8205,
"field_grounded_n": 39,
"empty": 0.0
},
"after_comma": {
"n": 150,
"match": 0.0133,
"suffix_consistent": 0.9933,
"balanced": 0.9933,
"op_valid": 0.9933,
"op_valid_n": 150,
"field_grounded": 0.9333,
"field_grounded_n": 150,
"empty": 0.0
},
"after_dollar": {
"n": 150,
"match": 0.0,
"suffix_consistent": 0.7933,
"balanced": 0.9133,
"op_valid": 0.7933,
"op_valid_n": 150,
"field_grounded": 0.8358,
"field_grounded_n": 67,
"empty": 0.0
},
"after_open": {
"n": 150,
"match": 0.0467,
"suffix_consistent": 0.9933,
"balanced": 0.9933,
"op_valid": 0.9933,
"op_valid_n": 150,
"field_grounded": 0.9429,
"field_grounded_n": 140,
"empty": 0.0
},
"dotted_path": {
"n": 150,
"match": 0.0333,
"suffix_consistent": 0.9933,
"balanced": 0.9933,
"op_valid": 0.9933,
"op_valid_n": 150,
"field_grounded": 0.88,
"field_grounded_n": 150,
"empty": 0.0
},
"in_string": {
"n": 150,
"match": 0.1267,
"suffix_consistent": 1.0,
"balanced": 1.0,
"op_valid": 0.9933,
"op_valid_n": 150,
"field_grounded": 0.8889,
"field_grounded_n": 18,
"empty": 0.0
},
"partial_ident": {
"n": 150,
"match": 0.1267,
"suffix_consistent": 0.9933,
"balanced": 0.9933,
"op_valid": 0.9933,
"op_valid_n": 150,
"field_grounded": 0.9067,
"field_grounded_n": 150,
"empty": 0.0
},
"partial_op": {
"n": 150,
"match": 0.1333,
"suffix_consistent": 0.84,
"balanced": 0.94,
"op_valid": 0.84,
"op_valid_n": 150,
"field_grounded": 0.9394,
"field_grounded_n": 66,
"empty": 0.0
}
},
"by_context_kind_at_64": {
"full": {
"n": 736,
"match": 0.0883,
"suffix_consistent": 0.9416,
"balanced": 0.9742,
"op_valid": 0.9416,
"op_valid_n": 736,
"field_grounded": 0.9917,
"field_grounded_n": 484,
"empty": 0.0
},
"irrelevant": {
"n": 244,
"match": 0.0738,
"suffix_consistent": 0.9467,
"balanced": 0.9795,
"op_valid": 0.9426,
"op_valid_n": 244,
"field_grounded": 0.9875,
"field_grounded_n": 160,
"empty": 0.0
},
"none": {
"n": 220,
"match": 0.0545,
"suffix_consistent": 0.9773,
"balanced": 0.9864,
"op_valid": 0.9773,
"op_valid_n": 220,
"field_grounded": 0.5,
"field_grounded_n": 136,
"empty": 0.0
}
},
"by_target_distance_at_64": {}
},
"t1_v4b_8k": {
"context_tokens": 8192,
"items": 1320,
"postprocess": "v2-close-inner",
"overall_at_64": {
"n": 1320,
"match": 0.0833,
"suffix_consistent": 0.9492,
"balanced": 0.9788,
"op_valid": 0.9485,
"op_valid_n": 1320,
"field_grounded": 0.9107,
"field_grounded_n": 851,
"empty": 0.0
},
"by_cut_kind_at_64": {
"after_colon": {
"n": 162,
"match": 0.179,
"suffix_consistent": 0.9877,
"balanced": 0.9938,
"op_valid": 0.9877,
"op_valid_n": 162,
"field_grounded": 0.825,
"field_grounded_n": 40,
"empty": 0.0
},
"after_comma": {
"n": 168,
"match": 0.0119,
"suffix_consistent": 0.994,
"balanced": 0.994,
"op_valid": 0.994,
"op_valid_n": 168,
"field_grounded": 0.9405,
"field_grounded_n": 168,
"empty": 0.0
},
"after_dollar": {
"n": 163,
"match": 0.0,
"suffix_consistent": 0.7914,
"balanced": 0.9141,
"op_valid": 0.7914,
"op_valid_n": 163,
"field_grounded": 0.8333,
"field_grounded_n": 72,
"empty": 0.0
},
"after_open": {
"n": 165,
"match": 0.0424,
"suffix_consistent": 0.9939,
"balanced": 0.9939,
"op_valid": 0.9939,
"op_valid_n": 165,
"field_grounded": 0.9481,
"field_grounded_n": 154,
"empty": 0.0
},
"dotted_path": {
"n": 159,
"match": 0.0314,
"suffix_consistent": 0.9937,
"balanced": 0.9937,
"op_valid": 0.9937,
"op_valid_n": 159,
"field_grounded": 0.8868,
"field_grounded_n": 159,
"empty": 0.0
},
"in_string": {
"n": 168,
"match": 0.131,
"suffix_consistent": 1.0,
"balanced": 1.0,
"op_valid": 0.994,
"op_valid_n": 168,
"field_grounded": 0.8889,
"field_grounded_n": 18,
"empty": 0.0
},
"partial_ident": {
"n": 169,
"match": 0.1302,
"suffix_consistent": 0.9941,
"balanced": 0.9941,
"op_valid": 0.9941,
"op_valid_n": 169,
"field_grounded": 0.9112,
"field_grounded_n": 169,
"empty": 0.0
},
"partial_op": {
"n": 166,
"match": 0.1386,
"suffix_consistent": 0.8373,
"balanced": 0.9458,
"op_valid": 0.8373,
"op_valid_n": 166,
"field_grounded": 0.9437,
"field_grounded_n": 71,
"empty": 0.0
}
},
"by_context_kind_at_64": {
"full": {
"n": 736,
"match": 0.0883,
"suffix_consistent": 0.9416,
"balanced": 0.9742,
"op_valid": 0.9416,
"op_valid_n": 736,
"field_grounded": 0.9917,
"field_grounded_n": 484,
"empty": 0.0
},
"irrelevant": {
"n": 244,
"match": 0.0738,
"suffix_consistent": 0.9467,
"balanced": 0.9795,
"op_valid": 0.9426,
"op_valid_n": 244,
"field_grounded": 0.9875,
"field_grounded_n": 160,
"empty": 0.0
},
"long": {
"n": 120,
"match": 0.125,
"suffix_consistent": 0.95,
"balanced": 0.9917,
"op_valid": 0.95,
"op_valid_n": 120,
"field_grounded": 0.9718,
"field_grounded_n": 71,
"empty": 0.0
},
"none": {
"n": 220,
"match": 0.0545,
"suffix_consistent": 0.9773,
"balanced": 0.9864,
"op_valid": 0.9773,
"op_valid_n": 220,
"field_grounded": 0.5,
"field_grounded_n": 136,
"empty": 0.0
}
},
"by_target_distance_at_64": {
"2048": {
"n": 60,
"match": 0.1333,
"suffix_consistent": 0.9667,
"balanced": 1.0,
"op_valid": 0.9667,
"op_valid_n": 60,
"field_grounded": 0.973,
"field_grounded_n": 37,
"empty": 0.0
},
"6144": {
"n": 60,
"match": 0.1167,
"suffix_consistent": 0.9333,
"balanced": 0.9833,
"op_valid": 0.9333,
"op_valid_n": 60,
"field_grounded": 0.9706,
"field_grounded_n": 34,
"empty": 0.0
}
}
}
},
"definitions": {
"suffix_consistent": "prefix + suggestion + suffix parses (no duplicated or missing closer)",
"op_valid": "the document parses and every operator is legal where it is",
"field_grounded": "every field the suggestion cites exists in the target collection schema",
"postprocess": "editor-context-v2 output rule: missing inner closers are appended when the suffix only closes the outer ones"
}
}