Jeeves v2: SFT on short, finished reasoning chains, public data only, two temperatures
27c0c79 verified Download export.json from PostHog/jeeves: direct link, hf CLI and curl.
- Browser
- Download file 5.89 kB
-
https://huggingface.co/PostHog/jeeves/resolve/main/export.json
- Command line
-
hf download hf://PostHog/jeeves/export.json
-
curl -L -o export.json https://huggingface.co/PostHog/jeeves/resolve/main/export.json
5.89 kB
| { | |
| "source_checkpoint": "runs/sft/final", | |
| "base_model": "Qwen/Qwen3.5-9B", | |
| "dtype": "bfloat16", | |
| "format": "markers-v3-plainchains", | |
| "head_dim": 256, | |
| "temperature": 1.522, | |
| "think_temperature": 5.4195, | |
| "stage": "sft", | |
| "step": 16867, | |
| "sources": { | |
| "banking77": 0, | |
| "boolq": 1, | |
| "agnews": 2, | |
| "mnli": 3, | |
| "sst5": 4, | |
| "yelp": 5, | |
| "trec": 6, | |
| "dbpedia14": 7, | |
| "amazon": 8, | |
| "imdb": 9, | |
| "mmlu": 10, | |
| "emotion": 11, | |
| "tweet_offensive": 12, | |
| "qnli": 13, | |
| "paws": 14, | |
| "sciq": 15, | |
| "contrastive": 16, | |
| "compositional": 17, | |
| "composition_holdout": 18, | |
| "semif": 19, | |
| "wanli": 20, | |
| "mmlu_pro": 21, | |
| "buried": 22, | |
| "unknowable": 23, | |
| "unknowable_control": 24, | |
| "aqua_rat": 25, | |
| "cosmos_qa": 26, | |
| "esci": 27, | |
| "glaive_fc": 28, | |
| "gsm8k": 29, | |
| "hellaswag": 30, | |
| "newyorker": 31, | |
| "openbookqa": 32, | |
| "ragtruth": 33, | |
| "ruletaker": 34, | |
| "sgd": 35, | |
| "swag": 36, | |
| "toolace": 37, | |
| "when2call": 38, | |
| "winogrande": 39, | |
| "decider_qasc": 40, | |
| "decider_massive_intent_upstream": 41, | |
| "night2_dates": 42, | |
| "kev/documents-v1": 43, | |
| "decider_boolq": 44, | |
| "decider/routing_messages": 45, | |
| "decider/commands": 46, | |
| "decider_go_emotions": 47, | |
| "decider_banking77": 48, | |
| "typed_decisions": 49, | |
| "decider_commonsense_qa": 50, | |
| "pi_llmail": 51, | |
| "decider_helpsteer3_pref": 52, | |
| "decider_medqa": 53, | |
| "decider_prosocial_safety": 54, | |
| "openjev/v1/phone-extraction-control-v1": 55, | |
| "kev/hard-v1": 56, | |
| "decider_dbpedia": 57, | |
| "decider_hh_rlhf": 58, | |
| "decider_bias_in_bios": 59, | |
| "openjev/v1/mailroom-control-v1": 60, | |
| "decider_json_state": 61, | |
| "decider_rules": 62, | |
| "pi_slabs": 63, | |
| "openjev/v1/silent-failure-control-v1": 64, | |
| "decider_medmcqa": 65, | |
| "decider_mind2web": 66, | |
| "pi_neuralchemy_v1": 67, | |
| "openjev/v1/amount-extraction-control-v1": 68, | |
| "decider/routing_terse": 69, | |
| "openjev/v1/browser-drone-expansion-v1-redistributable": 70, | |
| "openjev/v1/ir-control-v1": 71, | |
| "pi_gandalf": 72, | |
| "decider_snli": 73, | |
| "decider_arc": 74, | |
| "pi_harelix": 75, | |
| "openjev/v1/email-selection-control-v1": 76, | |
| "decider_clinc_oos": 77, | |
| "openjev/v1.1/community-hard-mix-v2-redistributable": 78, | |
| "pi_hackaprompt": 79, | |
| "pi_mixP_benign": 80, | |
| "pi_injecagent": 81, | |
| "night2_assertion": 82, | |
| "decider/situations": 83, | |
| "kev/devtools-v1": 84, | |
| "decider_civil_comments": 85, | |
| "pi_alespalla": 86, | |
| "decider_hate_speech_scales": 87, | |
| "decider_liar2": 88, | |
| "pi_yanismiraoui": 89, | |
| "decider_isolated": 90, | |
| "decider_helpsteer2": 91, | |
| "decider/custom_questions": 92, | |
| "openjev/v1/entity-alignment-control-v1": 93, | |
| "pi_harmbench_benign": 94, | |
| "spark": 95, | |
| "night2_unknowable": 96, | |
| "pi_jailbreakbench": 97, | |
| "night2_unknowable_control": 98, | |
| "openjev/v1/context-retention-control-v1": 99, | |
| "openjev/v1/citation-control-v1": 100, | |
| "pi_neuralchemy_original": 101, | |
| "pi_llmail_fp": 102, | |
| "openjev/v1/sponsor-segment-control-v1": 103 | |
| }, | |
| "metrics": { | |
| "dev_acc": 0.8311879367414491, | |
| "dev_nll": 0.4963217283180857, | |
| "dev_n": 5438, | |
| "per_source": { | |
| "banking77": 0.7759, | |
| "boolq": 0.95, | |
| "agnews": 0.9567, | |
| "mnli": 0.8017, | |
| "sst5": 0.6625, | |
| "yelp": 0.7688, | |
| "trec": 0.8448, | |
| "dbpedia14": 0.9741, | |
| "amazon": 0.65, | |
| "imdb": 0.975, | |
| "mmlu": 0.7328, | |
| "emotion": 0.5948, | |
| "tweet_offensive": 0.75, | |
| "qnli": 0.9375, | |
| "paws": 0.825, | |
| "sciq": 0.9655, | |
| "contrastive": 0.9694, | |
| "compositional": 0.9688, | |
| "composition_holdout": 0.9583, | |
| "semif": 0.8889, | |
| "wanli": 0.7073, | |
| "mmlu_pro": 0.6164, | |
| "buried": 0.7414, | |
| "unknowable": 0.4128, | |
| "unknowable_control": 0.9795, | |
| "aqua_rat": 0.6181, | |
| "cosmos_qa": 0.9, | |
| "esci": 0.5933, | |
| "glaive_fc": 0.9933, | |
| "gsm8k": 0.755, | |
| "hellaswag": 0.9536, | |
| "newyorker": 0.7415, | |
| "openbookqa": 0.8933, | |
| "ragtruth": 0.7933, | |
| "ruletaker": 0.94, | |
| "sgd": 0.9733, | |
| "swag": 0.8267, | |
| "toolace": 0.9533, | |
| "when2call": 0.9467, | |
| "winogrande": 0.92 | |
| }, | |
| "unknowable": { | |
| "n": 109, | |
| "mean_max_p": 0.5008, | |
| "share_at_0_9": 0.0183, | |
| "control_acc": 0.9795, | |
| "control_mean_max_p": 0.9638, | |
| "control_share_at_0_9": 0.8699 | |
| } | |
| }, | |
| "training_config": { | |
| "model": "Qwen/Qwen3.5-9B", | |
| "init": null, | |
| "lora_r": 16, | |
| "lora_alpha": 32.0, | |
| "head_dim": 256, | |
| "eval_batch_size": 16, | |
| "eval_think_limit": 256, | |
| "seed": 0, | |
| "max_think": 2560, | |
| "temperature": 1.0, | |
| "gen_batch_size": 64, | |
| "pad_multiple": 256, | |
| "num_workers": 2, | |
| "compile": true, | |
| "stage": "sft", | |
| "run_dir": "runs/sft", | |
| "data_dir": "data", | |
| "resume": null, | |
| "lr": 5e-05, | |
| "head_lr": 0.0001, | |
| "warmup": 20, | |
| "sft_epochs": 1, | |
| "cispo_epochs": 4, | |
| "stop_step": 402, | |
| "batch_size": 8, | |
| "grad_clip": 1.0, | |
| "max_len": 4096, | |
| "max_seq_len": 8192, | |
| "train_limit": 0, | |
| "dev_limit": 0, | |
| "eval_every": 2000, | |
| "eval_think": true, | |
| "log_every": 10, | |
| "gradient_checkpointing": true, | |
| "prompts_per_step": 8, | |
| "group": 8, | |
| "eps_high": 0.2, | |
| "rollout_ce_weight": 0.5, | |
| "anchor_ce_weight": 0.5, | |
| "anchor_batch": 8, | |
| "anchor_limit": 0, | |
| "logprob_chunk": 128, | |
| "micro_groups": 2, | |
| "length_hinge": 2048, | |
| "length_hinge_end": 1024, | |
| "length_penalty_cap": 0.1, | |
| "ckpt_minutes": 45.0, | |
| "log_rollouts": 3, | |
| "log_chain_chars": 1200, | |
| "sft_think_frac": 0.15, | |
| "chain_cache": "data/chains", | |
| "bucket": true, | |
| "chain_max_len": 1536, | |
| "unclosed_reward": 1.0, | |
| "loop_penalty": 0.0, | |
| "greedy_rollouts": 0, | |
| "chain_lm_weight": 0.5, | |
| "sft_truncate_frac": 0.2 | |
| } | |
| } | |