Text Generation
Transformers
Safetensors
English
Portuguese
qwen2
mongodb
code
fim
autocomplete
synthetic-data
conversational
Eval Results (legacy)
text-generation-inference
Instructions to use esilva/KapiCoder-Mongo-Small with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use esilva/KapiCoder-Mongo-Small with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="esilva/KapiCoder-Mongo-Small") messages = [ {"role": "user", "content": "Who are you?"}, ] pipe(messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("esilva/KapiCoder-Mongo-Small") model = AutoModelForCausalLM.from_pretrained("esilva/KapiCoder-Mongo-Small", device_map="auto") messages = [ {"role": "user", "content": "Who are you?"}, ] inputs = tokenizer.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use esilva/KapiCoder-Mongo-Small with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "esilva/KapiCoder-Mongo-Small" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "esilva/KapiCoder-Mongo-Small", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker
docker model run hf.co/esilva/KapiCoder-Mongo-Small
- SGLang
How to use esilva/KapiCoder-Mongo-Small with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "esilva/KapiCoder-Mongo-Small" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "esilva/KapiCoder-Mongo-Small", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "esilva/KapiCoder-Mongo-Small" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "esilva/KapiCoder-Mongo-Small", "messages": [ { "role": "user", "content": "What is the capital of France?" } ] }' - Docker Model Runner
How to use esilva/KapiCoder-Mongo-Small with Docker Model Runner:
docker model run hf.co/esilva/KapiCoder-Mongo-Small
Download evaluation/val_v2c_summary.json from esilva/KapiCoder-Mongo-Small: direct link, hf CLI and curl.
- Browser
- Download file 20.6 kB
-
https://huggingface.co/esilva/KapiCoder-Mongo-Small/resolve/main/evaluation/val_v2c_summary.json
- Command line
-
hf download hf://esilva/KapiCoder-Mongo-Small/evaluation/val_v2c_summary.json
-
curl -L -o val_v2c_summary.json https://huggingface.co/esilva/KapiCoder-Mongo-Small/resolve/main/evaluation/val_v2c_summary.json
20.6 kB
| { | |
| "schema": "kapi-hf-eval-summary/1", | |
| "kind": "val_v2c", | |
| "label": "Internal evaluation on synthetic data; no blind human-written set. Aggregates only; no item text.", | |
| "model": "esilva/KapiCoder-Mongo-Small", | |
| "split": "val", | |
| "n": 2000, | |
| "seed": 20261008, | |
| "contract": "editor-context-v1", | |
| "dataset": { | |
| "sample_ids_sha256": "28ba86be496839a321de0035d6a7e21827a5e8de4362151d80958ee8c798562f", | |
| "sha256": "af3331481966f3ebfb05d36109b43014cd850f669f6f5d2a524f8da6e9609ec7", | |
| "manifest_sha256": "2fef4f342d8a21eaf6d7e9eb1f223b3c8094d56c8dcd00e64f04bc20b20cd3cb", | |
| "stratify_by": [ | |
| "cut_kind", | |
| "domain" | |
| ], | |
| "cut_kind": { | |
| "multiline": 161, | |
| "key": 319, | |
| "value": 280, | |
| "rest": 241, | |
| "operator": 268, | |
| "member": 201, | |
| "stage": 251, | |
| "partial": 200, | |
| "stmt_start": 79 | |
| }, | |
| "domain": { | |
| "commerce": 168, | |
| "content": 161, | |
| "energy": 171, | |
| "finance": 175, | |
| "gaming": 163, | |
| "healthcare-synthetic": 173, | |
| "hr": 168, | |
| "iot": 168, | |
| "logistics": 157, | |
| "saas": 165, | |
| "travel": 162, | |
| "workspace": 169 | |
| } | |
| }, | |
| "decoding": "greedy (argmax), padding do vocab mascarado", | |
| "max_new_tokens": 96, | |
| "context_tokens": 2048, | |
| "load": "bf16", | |
| "this_model": { | |
| "headline": { | |
| "n": 2000, | |
| "exact_match": 0.399, | |
| "normalized_match": 0.399, | |
| "prefix_match": 0.4095, | |
| "edit_similarity": 0.6413, | |
| "chars_accepted": 7.8105, | |
| "tab_chars": 6.312, | |
| "accepted_prefix_tokens": 2.566, | |
| "syntax_valid": 0.9905, | |
| "mongo_valid": 0.9875, | |
| "exec_valid": 0.5815, | |
| "exec_valid_incl_empty": 0.5815, | |
| "nonexistent_fields_rate": 0.034, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.991, | |
| "mean_generated_tokens": 14.7685 | |
| }, | |
| "headline_ide32": { | |
| "n": 2000, | |
| "exact_match": 0.398, | |
| "normalized_match": 0.398, | |
| "prefix_match": 0.4095, | |
| "edit_similarity": 0.6355, | |
| "chars_accepted": 7.798, | |
| "tab_chars": 6.2995, | |
| "accepted_prefix_tokens": 2.5615, | |
| "syntax_valid": 0.864, | |
| "mongo_valid": 0.861, | |
| "exec_valid": 0.5095, | |
| "exec_valid_incl_empty": 0.5095, | |
| "nonexistent_fields_rate": 0.023, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": null, | |
| "mean_generated_tokens": null | |
| }, | |
| "reference_ceiling": { | |
| "syntax_valid": 1.0, | |
| "mongo_valid": 1.0, | |
| "exec_valid": 0.6255, | |
| "nonexistent_fields_rate": 0.0 | |
| }, | |
| "exec_stage_distribution": { | |
| "ok": 1163, | |
| "api": 480, | |
| "syntax": 258, | |
| "fields": 92, | |
| "exec": 3, | |
| "type": 4 | |
| }, | |
| "by_cut_kind": { | |
| "key": { | |
| "n": 319, | |
| "exact_match": 0.4357, | |
| "normalized_match": 0.4357, | |
| "prefix_match": 0.442, | |
| "edit_similarity": 0.6301, | |
| "chars_accepted": 6.1912, | |
| "tab_chars": 5.6207, | |
| "accepted_prefix_tokens": 1.9154, | |
| "syntax_valid": 0.9906, | |
| "mongo_valid": 0.9843, | |
| "exec_valid": 0.5204, | |
| "exec_valid_incl_empty": 0.5204, | |
| "nonexistent_fields_rate": 0.0502, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9937, | |
| "mean_generated_tokens": 8.4013 | |
| }, | |
| "member": { | |
| "n": 201, | |
| "exact_match": 0.7811, | |
| "normalized_match": 0.7811, | |
| "prefix_match": 0.7861, | |
| "edit_similarity": 0.8511, | |
| "chars_accepted": 8.3234, | |
| "tab_chars": 8.0597, | |
| "accepted_prefix_tokens": 1.7413, | |
| "syntax_valid": 1.0, | |
| "mongo_valid": 0.995, | |
| "exec_valid": 0.6368, | |
| "exec_valid_incl_empty": 0.6368, | |
| "nonexistent_fields_rate": 0.005, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 1.0, | |
| "mean_generated_tokens": 4.9701 | |
| }, | |
| "multiline": { | |
| "n": 161, | |
| "exact_match": 0.0, | |
| "normalized_match": 0.0, | |
| "prefix_match": 0.0, | |
| "edit_similarity": 0.3836, | |
| "chars_accepted": 9.528, | |
| "tab_chars": 5.8944, | |
| "accepted_prefix_tokens": 2.882, | |
| "syntax_valid": 0.9814, | |
| "mongo_valid": 0.9814, | |
| "exec_valid": 0.5839, | |
| "exec_valid_incl_empty": 0.5839, | |
| "nonexistent_fields_rate": 0.1118, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9814, | |
| "mean_generated_tokens": 41.6584 | |
| }, | |
| "operator": { | |
| "n": 268, | |
| "exact_match": 0.5896, | |
| "normalized_match": 0.5896, | |
| "prefix_match": 0.6045, | |
| "edit_similarity": 0.7908, | |
| "chars_accepted": 7.8657, | |
| "tab_chars": 6.5261, | |
| "accepted_prefix_tokens": 3.1082, | |
| "syntax_valid": 0.9925, | |
| "mongo_valid": 0.9888, | |
| "exec_valid": 0.6157, | |
| "exec_valid_incl_empty": 0.6157, | |
| "nonexistent_fields_rate": 0.0224, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9925, | |
| "mean_generated_tokens": 9.597 | |
| }, | |
| "partial": { | |
| "n": 200, | |
| "exact_match": 0.73, | |
| "normalized_match": 0.73, | |
| "prefix_match": 0.735, | |
| "edit_similarity": 0.8495, | |
| "chars_accepted": 6.075, | |
| "tab_chars": 5.375, | |
| "accepted_prefix_tokens": 2.3, | |
| "syntax_valid": 0.995, | |
| "mongo_valid": 0.99, | |
| "exec_valid": 0.585, | |
| "exec_valid_incl_empty": 0.585, | |
| "nonexistent_fields_rate": 0.035, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 1.0, | |
| "mean_generated_tokens": 7.175 | |
| }, | |
| "rest": { | |
| "n": 241, | |
| "exact_match": 0.3402, | |
| "normalized_match": 0.3402, | |
| "prefix_match": 0.3444, | |
| "edit_similarity": 0.6247, | |
| "chars_accepted": 9.2946, | |
| "tab_chars": 7.3237, | |
| "accepted_prefix_tokens": 2.9087, | |
| "syntax_valid": 0.9917, | |
| "mongo_valid": 0.9876, | |
| "exec_valid": 0.5685, | |
| "exec_valid_incl_empty": 0.5685, | |
| "nonexistent_fields_rate": 0.0166, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9959, | |
| "mean_generated_tokens": 14.7676 | |
| }, | |
| "stage": { | |
| "n": 251, | |
| "exact_match": 0.0159, | |
| "normalized_match": 0.0159, | |
| "prefix_match": 0.0159, | |
| "edit_similarity": 0.4148, | |
| "chars_accepted": 11.004, | |
| "tab_chars": 7.753, | |
| "accepted_prefix_tokens": 4.0637, | |
| "syntax_valid": 0.9761, | |
| "mongo_valid": 0.9761, | |
| "exec_valid": 0.5538, | |
| "exec_valid_incl_empty": 0.5538, | |
| "nonexistent_fields_rate": 0.0438, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9641, | |
| "mean_generated_tokens": 30.9323 | |
| }, | |
| "stmt_start": { | |
| "n": 79, | |
| "exact_match": 0.3165, | |
| "normalized_match": 0.3165, | |
| "prefix_match": 0.3165, | |
| "edit_similarity": 0.6021, | |
| "chars_accepted": 16.1772, | |
| "tab_chars": 14.5696, | |
| "accepted_prefix_tokens": 3.6709, | |
| "syntax_valid": 1.0, | |
| "mongo_valid": 1.0, | |
| "exec_valid": 0.6456, | |
| "exec_valid_incl_empty": 0.6456, | |
| "nonexistent_fields_rate": 0.0633, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 1.0, | |
| "mean_generated_tokens": 16.4304 | |
| }, | |
| "value": { | |
| "n": 280, | |
| "exact_match": 0.3107, | |
| "normalized_match": 0.3107, | |
| "prefix_match": 0.3536, | |
| "edit_similarity": 0.5887, | |
| "chars_accepted": 2.9857, | |
| "tab_chars": 2.0571, | |
| "accepted_prefix_tokens": 1.4393, | |
| "syntax_valid": 0.9929, | |
| "mongo_valid": 0.9929, | |
| "exec_valid": 0.5929, | |
| "exec_valid_incl_empty": 0.5929, | |
| "nonexistent_fields_rate": 0.0, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9964, | |
| "mean_generated_tokens": 9.0107 | |
| } | |
| }, | |
| "by_domain": { | |
| "commerce": { | |
| "n": 168, | |
| "exact_match": 0.381, | |
| "normalized_match": 0.381, | |
| "prefix_match": 0.3929, | |
| "edit_similarity": 0.6256, | |
| "chars_accepted": 7.1548, | |
| "tab_chars": 5.6012, | |
| "accepted_prefix_tokens": 2.256, | |
| "syntax_valid": 1.0, | |
| "mongo_valid": 0.9881, | |
| "exec_valid": 0.6607, | |
| "exec_valid_incl_empty": 0.6607, | |
| "nonexistent_fields_rate": 0.0119, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 1.0, | |
| "mean_generated_tokens": 14.875 | |
| }, | |
| "content": { | |
| "n": 161, | |
| "exact_match": 0.3665, | |
| "normalized_match": 0.3665, | |
| "prefix_match": 0.3851, | |
| "edit_similarity": 0.6143, | |
| "chars_accepted": 6.9876, | |
| "tab_chars": 5.6646, | |
| "accepted_prefix_tokens": 2.2795, | |
| "syntax_valid": 0.9938, | |
| "mongo_valid": 0.9876, | |
| "exec_valid": 0.5839, | |
| "exec_valid_incl_empty": 0.5839, | |
| "nonexistent_fields_rate": 0.0248, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9876, | |
| "mean_generated_tokens": 13.5528 | |
| }, | |
| "energy": { | |
| "n": 171, | |
| "exact_match": 0.4152, | |
| "normalized_match": 0.4152, | |
| "prefix_match": 0.4327, | |
| "edit_similarity": 0.654, | |
| "chars_accepted": 8.0409, | |
| "tab_chars": 6.883, | |
| "accepted_prefix_tokens": 2.4386, | |
| "syntax_valid": 0.9883, | |
| "mongo_valid": 0.9883, | |
| "exec_valid": 0.5848, | |
| "exec_valid_incl_empty": 0.5848, | |
| "nonexistent_fields_rate": 0.0526, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9825, | |
| "mean_generated_tokens": 15.6959 | |
| }, | |
| "finance": { | |
| "n": 175, | |
| "exact_match": 0.3543, | |
| "normalized_match": 0.3543, | |
| "prefix_match": 0.36, | |
| "edit_similarity": 0.604, | |
| "chars_accepted": 8.2286, | |
| "tab_chars": 6.8057, | |
| "accepted_prefix_tokens": 2.9086, | |
| "syntax_valid": 0.9829, | |
| "mongo_valid": 0.9829, | |
| "exec_valid": 0.5486, | |
| "exec_valid_incl_empty": 0.5486, | |
| "nonexistent_fields_rate": 0.0686, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9943, | |
| "mean_generated_tokens": 16.8743 | |
| }, | |
| "gaming": { | |
| "n": 163, | |
| "exact_match": 0.3681, | |
| "normalized_match": 0.3681, | |
| "prefix_match": 0.3804, | |
| "edit_similarity": 0.6173, | |
| "chars_accepted": 7.4356, | |
| "tab_chars": 6.0613, | |
| "accepted_prefix_tokens": 2.454, | |
| "syntax_valid": 0.9816, | |
| "mongo_valid": 0.9693, | |
| "exec_valid": 0.5828, | |
| "exec_valid_incl_empty": 0.5828, | |
| "nonexistent_fields_rate": 0.0368, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9816, | |
| "mean_generated_tokens": 15.5706 | |
| }, | |
| "healthcare-synthetic": { | |
| "n": 173, | |
| "exact_match": 0.3988, | |
| "normalized_match": 0.3988, | |
| "prefix_match": 0.4104, | |
| "edit_similarity": 0.6482, | |
| "chars_accepted": 8.341, | |
| "tab_chars": 6.3757, | |
| "accepted_prefix_tokens": 2.815, | |
| "syntax_valid": 0.9884, | |
| "mongo_valid": 0.9884, | |
| "exec_valid": 0.5723, | |
| "exec_valid_incl_empty": 0.5723, | |
| "nonexistent_fields_rate": 0.0116, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9827, | |
| "mean_generated_tokens": 14.5087 | |
| }, | |
| "hr": { | |
| "n": 168, | |
| "exact_match": 0.4643, | |
| "normalized_match": 0.4643, | |
| "prefix_match": 0.4762, | |
| "edit_similarity": 0.6813, | |
| "chars_accepted": 8.2976, | |
| "tab_chars": 7.0952, | |
| "accepted_prefix_tokens": 2.6607, | |
| "syntax_valid": 1.0, | |
| "mongo_valid": 1.0, | |
| "exec_valid": 0.5714, | |
| "exec_valid_incl_empty": 0.5714, | |
| "nonexistent_fields_rate": 0.0417, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 1.0, | |
| "mean_generated_tokens": 11.6488 | |
| }, | |
| "iot": { | |
| "n": 168, | |
| "exact_match": 0.4167, | |
| "normalized_match": 0.4167, | |
| "prefix_match": 0.4226, | |
| "edit_similarity": 0.6363, | |
| "chars_accepted": 8.0714, | |
| "tab_chars": 6.4048, | |
| "accepted_prefix_tokens": 2.6964, | |
| "syntax_valid": 1.0, | |
| "mongo_valid": 1.0, | |
| "exec_valid": 0.6369, | |
| "exec_valid_incl_empty": 0.6369, | |
| "nonexistent_fields_rate": 0.0476, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 1.0, | |
| "mean_generated_tokens": 15.1905 | |
| }, | |
| "logistics": { | |
| "n": 157, | |
| "exact_match": 0.4076, | |
| "normalized_match": 0.4076, | |
| "prefix_match": 0.4204, | |
| "edit_similarity": 0.6843, | |
| "chars_accepted": 8.2739, | |
| "tab_chars": 6.3503, | |
| "accepted_prefix_tokens": 2.7134, | |
| "syntax_valid": 1.0, | |
| "mongo_valid": 0.9936, | |
| "exec_valid": 0.5924, | |
| "exec_valid_incl_empty": 0.5924, | |
| "nonexistent_fields_rate": 0.0318, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 1.0, | |
| "mean_generated_tokens": 13.3312 | |
| }, | |
| "saas": { | |
| "n": 165, | |
| "exact_match": 0.4788, | |
| "normalized_match": 0.4788, | |
| "prefix_match": 0.4848, | |
| "edit_similarity": 0.6914, | |
| "chars_accepted": 7.3152, | |
| "tab_chars": 5.5939, | |
| "accepted_prefix_tokens": 2.497, | |
| "syntax_valid": 0.9758, | |
| "mongo_valid": 0.9758, | |
| "exec_valid": 0.4788, | |
| "exec_valid_incl_empty": 0.4788, | |
| "nonexistent_fields_rate": 0.0303, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9879, | |
| "mean_generated_tokens": 13.4485 | |
| }, | |
| "travel": { | |
| "n": 162, | |
| "exact_match": 0.3889, | |
| "normalized_match": 0.3889, | |
| "prefix_match": 0.3951, | |
| "edit_similarity": 0.6216, | |
| "chars_accepted": 7.8272, | |
| "tab_chars": 6.4383, | |
| "accepted_prefix_tokens": 2.6173, | |
| "syntax_valid": 1.0, | |
| "mongo_valid": 1.0, | |
| "exec_valid": 0.6296, | |
| "exec_valid_incl_empty": 0.6296, | |
| "nonexistent_fields_rate": 0.0247, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 1.0, | |
| "mean_generated_tokens": 15.7037 | |
| }, | |
| "workspace": { | |
| "n": 169, | |
| "exact_match": 0.3491, | |
| "normalized_match": 0.3491, | |
| "prefix_match": 0.355, | |
| "edit_similarity": 0.6204, | |
| "chars_accepted": 7.6923, | |
| "tab_chars": 6.3964, | |
| "accepted_prefix_tokens": 2.432, | |
| "syntax_valid": 0.9763, | |
| "mongo_valid": 0.9763, | |
| "exec_valid": 0.5385, | |
| "exec_valid_incl_empty": 0.5385, | |
| "nonexistent_fields_rate": 0.0237, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9763, | |
| "mean_generated_tokens": 16.6036 | |
| } | |
| }, | |
| "by_context_kind": { | |
| "full": { | |
| "n": 1647, | |
| "exact_match": 0.408, | |
| "normalized_match": 0.408, | |
| "prefix_match": 0.4189, | |
| "edit_similarity": 0.6525, | |
| "chars_accepted": 8.0291, | |
| "tab_chars": 6.5209, | |
| "accepted_prefix_tokens": 2.6436, | |
| "syntax_valid": 0.9897, | |
| "mongo_valid": 0.9879, | |
| "exec_valid": 0.6072, | |
| "exec_valid_incl_empty": 0.6072, | |
| "nonexistent_fields_rate": 0.0182, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9909, | |
| "mean_generated_tokens": 14.8336 | |
| }, | |
| "irrelevant": { | |
| "n": 198, | |
| "exact_match": 0.3636, | |
| "normalized_match": 0.3636, | |
| "prefix_match": 0.3687, | |
| "edit_similarity": 0.584, | |
| "chars_accepted": 7.1111, | |
| "tab_chars": 5.8232, | |
| "accepted_prefix_tokens": 2.2222, | |
| "syntax_valid": 0.9899, | |
| "mongo_valid": 0.9798, | |
| "exec_valid": 0.4747, | |
| "exec_valid_incl_empty": 0.4747, | |
| "nonexistent_fields_rate": 0.096, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9899, | |
| "mean_generated_tokens": 14.5354 | |
| }, | |
| "none": { | |
| "n": 155, | |
| "exact_match": 0.3484, | |
| "normalized_match": 0.3484, | |
| "prefix_match": 0.3613, | |
| "edit_similarity": 0.5964, | |
| "chars_accepted": 6.3806, | |
| "tab_chars": 4.7161, | |
| "accepted_prefix_tokens": 2.1806, | |
| "syntax_valid": 1.0, | |
| "mongo_valid": 0.9935, | |
| "exec_valid": 0.4452, | |
| "exec_valid_incl_empty": 0.4452, | |
| "nonexistent_fields_rate": 0.1226, | |
| "empty_rate": 0.0, | |
| "eos_or_suffix_stop_rate": 0.9935, | |
| "mean_generated_tokens": 14.3742 | |
| } | |
| } | |
| }, | |
| "metrics_by_model": { | |
| "this_model_v4b": { | |
| "syntax_valid": 0.9905, | |
| "mongodb_valid": 0.9875, | |
| "nonexistent_fields_rate": 3.4, | |
| "conversational_suggestion_rate": 0.0, | |
| "fim_acc": 0.399, | |
| "exact_match": 0.399, | |
| "apt_mean": 2.566, | |
| "exec_valid_autocomplete": 0.5815, | |
| "empty_rate": 0.0, | |
| "reference_ceiling_exec_valid": 0.6255, | |
| "exec_valid_vs_ceiling": 0.9297, | |
| "nonexistent_fields_full_rate_pct": 1.82, | |
| "n_context_full": 1647, | |
| "exec_valid_full": 0.6072, | |
| "nonexistent_fields_irrelevant_rate_pct": 9.6, | |
| "n_context_irrelevant": 198, | |
| "exec_valid_irrelevant": 0.4747, | |
| "nonexistent_fields_none_rate_pct": 12.26, | |
| "n_context_none": 155, | |
| "exec_valid_none": 0.4452, | |
| "info_fim_acc_gain_vs_zero_shot_base_pp": 25.7, | |
| "info_apt_delta_vs_zero_shot_base": 1.127, | |
| "info_exec_valid_delta_vs_zero_shot_base_pp": 26.55, | |
| "generic_js_fim_acc": 0.626, | |
| "generic_js_fim_delta_pp_vs_zero_shot_base": 22.0, | |
| "write_or_destructive_executions": 0, | |
| "train_eval_leak_count": 0, | |
| "decontamination_approved": true, | |
| "auto_report_attached": true | |
| }, | |
| "previous_release_v4_v0.2.0_preview": { | |
| "syntax_valid": 0.99, | |
| "mongodb_valid": 0.987, | |
| "nonexistent_fields_rate": 3.75, | |
| "conversational_suggestion_rate": 0.0, | |
| "fim_acc": 0.4065, | |
| "exact_match": 0.4065, | |
| "apt_mean": 2.6265, | |
| "exec_valid_autocomplete": 0.5745, | |
| "empty_rate": 0.0, | |
| "reference_ceiling_exec_valid": 0.6255, | |
| "exec_valid_vs_ceiling": 0.9185, | |
| "nonexistent_fields_full_rate_pct": 2.0, | |
| "n_context_full": 1647, | |
| "exec_valid_full": 0.5981, | |
| "nonexistent_fields_irrelevant_rate_pct": 12.63, | |
| "n_context_irrelevant": 198, | |
| "exec_valid_irrelevant": 0.4596, | |
| "nonexistent_fields_none_rate_pct": 10.97, | |
| "n_context_none": 155, | |
| "exec_valid_none": 0.471, | |
| "info_fim_acc_gain_vs_zero_shot_base_pp": 26.45, | |
| "info_apt_delta_vs_zero_shot_base": 1.1875, | |
| "info_exec_valid_delta_vs_zero_shot_base_pp": 25.85, | |
| "generic_js_fim_acc": 0.628, | |
| "generic_js_fim_delta_pp_vs_zero_shot_base": 22.2, | |
| "write_or_destructive_executions": 0, | |
| "train_eval_leak_count": 0, | |
| "decontamination_approved": true, | |
| "auto_report_attached": true | |
| }, | |
| "zero_shot_base": { | |
| "syntax_valid": 0.667, | |
| "mongodb_valid": 0.595, | |
| "nonexistent_fields_rate": 5.1, | |
| "conversational_suggestion_rate": 0.0, | |
| "fim_acc": 0.142, | |
| "exact_match": 0.142, | |
| "apt_mean": 1.439, | |
| "exec_valid_autocomplete": 0.316, | |
| "empty_rate": 0.0095, | |
| "reference_ceiling_exec_valid": 0.6255, | |
| "exec_valid_vs_ceiling": 0.5052, | |
| "nonexistent_fields_full_rate_pct": 3.76, | |
| "n_context_full": 1647, | |
| "exec_valid_full": 0.3364, | |
| "nonexistent_fields_irrelevant_rate_pct": 10.61, | |
| "n_context_irrelevant": 198, | |
| "exec_valid_irrelevant": 0.2172, | |
| "nonexistent_fields_none_rate_pct": 12.26, | |
| "n_context_none": 155, | |
| "exec_valid_none": 0.2258, | |
| "generic_js_fim_acc": 0.406, | |
| "write_or_destructive_executions": 0, | |
| "train_eval_leak_count": 0, | |
| "decontamination_approved": true, | |
| "auto_report_attached": true | |
| } | |
| }, | |
| "metric_definitions": { | |
| "exact_match": "pred.strip() == middle.strip()", | |
| "normalized_match": "igualdade com espacos normalizados (FIM accuracy do bake-off)", | |
| "chars_accepted": "tamanho do prefixo comum (caracteres) entre predicao e middle", | |
| "accepted_prefix_tokens": "tokens iniciais da referencia cobertos pelo prefixo comum (APT)", | |
| "syntax_valid": "node vm.Script sobre prefix+pred+suffix", | |
| "mongo_valid": "mongo_validate (AST, estatico) sobre o documento reconstruido", | |
| "exec_valid": "chat_validate.validate_text stage=ok (sintaxe, API do Console, campos, tipos, execucao somente-leitura em kapi_fx_*) E predicao nao vazia; exec_valid_incl_empty conta vazio como valido", | |
| "nonexistent_fields_rate": "stage=fields com 'campo/estrutura invalido' (campo ausente do schema) sobre todos os exemplos", | |
| "reference_ceiling": "mesmas checagens aplicadas ao middle de referencia (teto/calibracao do validador)" | |
| } | |
| } | |