{ "schema": "kapi-hf-eval-summary/1", "kind": "cursor_in_call_v1", "label": "Internal evaluation on synthetic data; no blind human-written set. Aggregates only; no item text.", "model": "esilva/KapiCoder-Mongo-Small", "suite_sha256_prefix": "e7b40a166c74", "decoding": "greedy", "limit_tokens": 64, "runs": { "t1_v3_ctxv2": { "context_tokens": 2048, "items": 1200, "postprocess": "v2-close-inner", "overall_at_64": { "n": 1200, "match": 0.065, "suffix_consistent": 0.8783, "balanced": 0.9383, "op_valid": 0.8567, "op_valid_n": 1200, "field_grounded": 0.8605, "field_grounded_n": 803, "empty": 0.0 }, "by_cut_kind_at_64": { "after_colon": { "n": 150, "match": 0.1667, "suffix_consistent": 0.9933, "balanced": 0.9933, "op_valid": 0.9733, "op_valid_n": 150, "field_grounded": 0.675, "field_grounded_n": 40, "empty": 0.0 }, "after_comma": { "n": 150, "match": 0.0, "suffix_consistent": 0.9933, "balanced": 1.0, "op_valid": 0.9867, "op_valid_n": 150, "field_grounded": 0.9267, "field_grounded_n": 150, "empty": 0.0 }, "after_dollar": { "n": 150, "match": 0.0, "suffix_consistent": 0.8133, "balanced": 0.9333, "op_valid": 0.8067, "op_valid_n": 150, "field_grounded": 0.8571, "field_grounded_n": 63, "empty": 0.0 }, "after_open": { "n": 150, "match": 0.0467, "suffix_consistent": 0.9933, "balanced": 0.9933, "op_valid": 0.98, "op_valid_n": 150, "field_grounded": 0.922, "field_grounded_n": 141, "empty": 0.0 }, "dotted_path": { "n": 150, "match": 0.0067, "suffix_consistent": 0.9933, "balanced": 0.9933, "op_valid": 0.9867, "op_valid_n": 150, "field_grounded": 0.88, "field_grounded_n": 150, "empty": 0.0 }, "in_string": { "n": 150, "match": 0.0733, "suffix_consistent": 0.4133, "balanced": 0.66, "op_valid": 0.4067, "op_valid_n": 150, "field_grounded": 0.8444, "field_grounded_n": 45, "empty": 0.0 }, "partial_ident": { "n": 150, "match": 0.1333, "suffix_consistent": 0.9933, "balanced": 0.9933, "op_valid": 0.9267, "op_valid_n": 150, "field_grounded": 0.8067, "field_grounded_n": 150, "empty": 0.0 }, "partial_op": { "n": 150, "match": 0.0933, "suffix_consistent": 0.8333, "balanced": 0.94, "op_valid": 0.7867, "op_valid_n": 150, "field_grounded": 0.7812, "field_grounded_n": 64, "empty": 0.0 } }, "by_context_kind_at_64": { "full": { "n": 736, "match": 0.0788, "suffix_consistent": 0.8696, "balanced": 0.928, "op_valid": 0.8465, "op_valid_n": 736, "field_grounded": 0.9614, "field_grounded_n": 492, "empty": 0.0 }, "irrelevant": { "n": 244, "match": 0.0451, "suffix_consistent": 0.873, "balanced": 0.9508, "op_valid": 0.8402, "op_valid_n": 244, "field_grounded": 0.8373, "field_grounded_n": 166, "empty": 0.0 }, "none": { "n": 220, "match": 0.0409, "suffix_consistent": 0.9136, "balanced": 0.9591, "op_valid": 0.9091, "op_valid_n": 220, "field_grounded": 0.5448, "field_grounded_n": 145, "empty": 0.0 } }, "by_target_distance_at_64": {} }, "t1_v4b_ctxv2": { "context_tokens": 2048, "items": 1200, "postprocess": "v2-close-inner", "overall_at_64": { "n": 1200, "match": 0.0792, "suffix_consistent": 0.9492, "balanced": 0.9775, "op_valid": 0.9483, "op_valid_n": 1200, "field_grounded": 0.9051, "field_grounded_n": 780, "empty": 0.0 }, "by_cut_kind_at_64": { "after_colon": { "n": 150, "match": 0.1533, "suffix_consistent": 0.9867, "balanced": 0.9933, "op_valid": 0.9867, "op_valid_n": 150, "field_grounded": 0.8205, "field_grounded_n": 39, "empty": 0.0 }, "after_comma": { "n": 150, "match": 0.0133, "suffix_consistent": 0.9933, "balanced": 0.9933, "op_valid": 0.9933, "op_valid_n": 150, "field_grounded": 0.9333, "field_grounded_n": 150, "empty": 0.0 }, "after_dollar": { "n": 150, "match": 0.0, "suffix_consistent": 0.7933, "balanced": 0.9133, "op_valid": 0.7933, "op_valid_n": 150, "field_grounded": 0.8358, "field_grounded_n": 67, "empty": 0.0 }, "after_open": { "n": 150, "match": 0.0467, "suffix_consistent": 0.9933, "balanced": 0.9933, "op_valid": 0.9933, "op_valid_n": 150, "field_grounded": 0.9429, "field_grounded_n": 140, "empty": 0.0 }, "dotted_path": { "n": 150, "match": 0.0333, "suffix_consistent": 0.9933, "balanced": 0.9933, "op_valid": 0.9933, "op_valid_n": 150, "field_grounded": 0.88, "field_grounded_n": 150, "empty": 0.0 }, "in_string": { "n": 150, "match": 0.1267, "suffix_consistent": 1.0, "balanced": 1.0, "op_valid": 0.9933, "op_valid_n": 150, "field_grounded": 0.8889, "field_grounded_n": 18, "empty": 0.0 }, "partial_ident": { "n": 150, "match": 0.1267, "suffix_consistent": 0.9933, "balanced": 0.9933, "op_valid": 0.9933, "op_valid_n": 150, "field_grounded": 0.9067, "field_grounded_n": 150, "empty": 0.0 }, "partial_op": { "n": 150, "match": 0.1333, "suffix_consistent": 0.84, "balanced": 0.94, "op_valid": 0.84, "op_valid_n": 150, "field_grounded": 0.9394, "field_grounded_n": 66, "empty": 0.0 } }, "by_context_kind_at_64": { "full": { "n": 736, "match": 0.0883, "suffix_consistent": 0.9416, "balanced": 0.9742, "op_valid": 0.9416, "op_valid_n": 736, "field_grounded": 0.9917, "field_grounded_n": 484, "empty": 0.0 }, "irrelevant": { "n": 244, "match": 0.0738, "suffix_consistent": 0.9467, "balanced": 0.9795, "op_valid": 0.9426, "op_valid_n": 244, "field_grounded": 0.9875, "field_grounded_n": 160, "empty": 0.0 }, "none": { "n": 220, "match": 0.0545, "suffix_consistent": 0.9773, "balanced": 0.9864, "op_valid": 0.9773, "op_valid_n": 220, "field_grounded": 0.5, "field_grounded_n": 136, "empty": 0.0 } }, "by_target_distance_at_64": {} }, "t1_v4b_8k": { "context_tokens": 8192, "items": 1320, "postprocess": "v2-close-inner", "overall_at_64": { "n": 1320, "match": 0.0833, "suffix_consistent": 0.9492, "balanced": 0.9788, "op_valid": 0.9485, "op_valid_n": 1320, "field_grounded": 0.9107, "field_grounded_n": 851, "empty": 0.0 }, "by_cut_kind_at_64": { "after_colon": { "n": 162, "match": 0.179, "suffix_consistent": 0.9877, "balanced": 0.9938, "op_valid": 0.9877, "op_valid_n": 162, "field_grounded": 0.825, "field_grounded_n": 40, "empty": 0.0 }, "after_comma": { "n": 168, "match": 0.0119, "suffix_consistent": 0.994, "balanced": 0.994, "op_valid": 0.994, "op_valid_n": 168, "field_grounded": 0.9405, "field_grounded_n": 168, "empty": 0.0 }, "after_dollar": { "n": 163, "match": 0.0, "suffix_consistent": 0.7914, "balanced": 0.9141, "op_valid": 0.7914, "op_valid_n": 163, "field_grounded": 0.8333, "field_grounded_n": 72, "empty": 0.0 }, "after_open": { "n": 165, "match": 0.0424, "suffix_consistent": 0.9939, "balanced": 0.9939, "op_valid": 0.9939, "op_valid_n": 165, "field_grounded": 0.9481, "field_grounded_n": 154, "empty": 0.0 }, "dotted_path": { "n": 159, "match": 0.0314, "suffix_consistent": 0.9937, "balanced": 0.9937, "op_valid": 0.9937, "op_valid_n": 159, "field_grounded": 0.8868, "field_grounded_n": 159, "empty": 0.0 }, "in_string": { "n": 168, "match": 0.131, "suffix_consistent": 1.0, "balanced": 1.0, "op_valid": 0.994, "op_valid_n": 168, "field_grounded": 0.8889, "field_grounded_n": 18, "empty": 0.0 }, "partial_ident": { "n": 169, "match": 0.1302, "suffix_consistent": 0.9941, "balanced": 0.9941, "op_valid": 0.9941, "op_valid_n": 169, "field_grounded": 0.9112, "field_grounded_n": 169, "empty": 0.0 }, "partial_op": { "n": 166, "match": 0.1386, "suffix_consistent": 0.8373, "balanced": 0.9458, "op_valid": 0.8373, "op_valid_n": 166, "field_grounded": 0.9437, "field_grounded_n": 71, "empty": 0.0 } }, "by_context_kind_at_64": { "full": { "n": 736, "match": 0.0883, "suffix_consistent": 0.9416, "balanced": 0.9742, "op_valid": 0.9416, "op_valid_n": 736, "field_grounded": 0.9917, "field_grounded_n": 484, "empty": 0.0 }, "irrelevant": { "n": 244, "match": 0.0738, "suffix_consistent": 0.9467, "balanced": 0.9795, "op_valid": 0.9426, "op_valid_n": 244, "field_grounded": 0.9875, "field_grounded_n": 160, "empty": 0.0 }, "long": { "n": 120, "match": 0.125, "suffix_consistent": 0.95, "balanced": 0.9917, "op_valid": 0.95, "op_valid_n": 120, "field_grounded": 0.9718, "field_grounded_n": 71, "empty": 0.0 }, "none": { "n": 220, "match": 0.0545, "suffix_consistent": 0.9773, "balanced": 0.9864, "op_valid": 0.9773, "op_valid_n": 220, "field_grounded": 0.5, "field_grounded_n": 136, "empty": 0.0 } }, "by_target_distance_at_64": { "2048": { "n": 60, "match": 0.1333, "suffix_consistent": 0.9667, "balanced": 1.0, "op_valid": 0.9667, "op_valid_n": 60, "field_grounded": 0.973, "field_grounded_n": 37, "empty": 0.0 }, "6144": { "n": 60, "match": 0.1167, "suffix_consistent": 0.9333, "balanced": 0.9833, "op_valid": 0.9333, "op_valid_n": 60, "field_grounded": 0.9706, "field_grounded_n": 34, "empty": 0.0 } } } }, "definitions": { "suffix_consistent": "prefix + suggestion + suffix parses (no duplicated or missing closer)", "op_valid": "the document parses and every operator is legal where it is", "field_grounded": "every field the suggestion cites exists in the target collection schema", "postprocess": "editor-context-v2 output rule: missing inner closers are appended when the suffix only closes the outer ones" } }