jeeves / export.json
rorylshanks's picture
Jeeves v2: SFT on short, finished reasoning chains, public data only, two temperatures
27c0c79 verified
Raw History Blame Contribute Delete
5.89 kB
{
"source_checkpoint": "runs/sft/final",
"base_model": "Qwen/Qwen3.5-9B",
"dtype": "bfloat16",
"format": "markers-v3-plainchains",
"head_dim": 256,
"temperature": 1.522,
"think_temperature": 5.4195,
"stage": "sft",
"step": 16867,
"sources": {
"banking77": 0,
"boolq": 1,
"agnews": 2,
"mnli": 3,
"sst5": 4,
"yelp": 5,
"trec": 6,
"dbpedia14": 7,
"amazon": 8,
"imdb": 9,
"mmlu": 10,
"emotion": 11,
"tweet_offensive": 12,
"qnli": 13,
"paws": 14,
"sciq": 15,
"contrastive": 16,
"compositional": 17,
"composition_holdout": 18,
"semif": 19,
"wanli": 20,
"mmlu_pro": 21,
"buried": 22,
"unknowable": 23,
"unknowable_control": 24,
"aqua_rat": 25,
"cosmos_qa": 26,
"esci": 27,
"glaive_fc": 28,
"gsm8k": 29,
"hellaswag": 30,
"newyorker": 31,
"openbookqa": 32,
"ragtruth": 33,
"ruletaker": 34,
"sgd": 35,
"swag": 36,
"toolace": 37,
"when2call": 38,
"winogrande": 39,
"decider_qasc": 40,
"decider_massive_intent_upstream": 41,
"night2_dates": 42,
"kev/documents-v1": 43,
"decider_boolq": 44,
"decider/routing_messages": 45,
"decider/commands": 46,
"decider_go_emotions": 47,
"decider_banking77": 48,
"typed_decisions": 49,
"decider_commonsense_qa": 50,
"pi_llmail": 51,
"decider_helpsteer3_pref": 52,
"decider_medqa": 53,
"decider_prosocial_safety": 54,
"openjev/v1/phone-extraction-control-v1": 55,
"kev/hard-v1": 56,
"decider_dbpedia": 57,
"decider_hh_rlhf": 58,
"decider_bias_in_bios": 59,
"openjev/v1/mailroom-control-v1": 60,
"decider_json_state": 61,
"decider_rules": 62,
"pi_slabs": 63,
"openjev/v1/silent-failure-control-v1": 64,
"decider_medmcqa": 65,
"decider_mind2web": 66,
"pi_neuralchemy_v1": 67,
"openjev/v1/amount-extraction-control-v1": 68,
"decider/routing_terse": 69,
"openjev/v1/browser-drone-expansion-v1-redistributable": 70,
"openjev/v1/ir-control-v1": 71,
"pi_gandalf": 72,
"decider_snli": 73,
"decider_arc": 74,
"pi_harelix": 75,
"openjev/v1/email-selection-control-v1": 76,
"decider_clinc_oos": 77,
"openjev/v1.1/community-hard-mix-v2-redistributable": 78,
"pi_hackaprompt": 79,
"pi_mixP_benign": 80,
"pi_injecagent": 81,
"night2_assertion": 82,
"decider/situations": 83,
"kev/devtools-v1": 84,
"decider_civil_comments": 85,
"pi_alespalla": 86,
"decider_hate_speech_scales": 87,
"decider_liar2": 88,
"pi_yanismiraoui": 89,
"decider_isolated": 90,
"decider_helpsteer2": 91,
"decider/custom_questions": 92,
"openjev/v1/entity-alignment-control-v1": 93,
"pi_harmbench_benign": 94,
"spark": 95,
"night2_unknowable": 96,
"pi_jailbreakbench": 97,
"night2_unknowable_control": 98,
"openjev/v1/context-retention-control-v1": 99,
"openjev/v1/citation-control-v1": 100,
"pi_neuralchemy_original": 101,
"pi_llmail_fp": 102,
"openjev/v1/sponsor-segment-control-v1": 103
},
"metrics": {
"dev_acc": 0.8311879367414491,
"dev_nll": 0.4963217283180857,
"dev_n": 5438,
"per_source": {
"banking77": 0.7759,
"boolq": 0.95,
"agnews": 0.9567,
"mnli": 0.8017,
"sst5": 0.6625,
"yelp": 0.7688,
"trec": 0.8448,
"dbpedia14": 0.9741,
"amazon": 0.65,
"imdb": 0.975,
"mmlu": 0.7328,
"emotion": 0.5948,
"tweet_offensive": 0.75,
"qnli": 0.9375,
"paws": 0.825,
"sciq": 0.9655,
"contrastive": 0.9694,
"compositional": 0.9688,
"composition_holdout": 0.9583,
"semif": 0.8889,
"wanli": 0.7073,
"mmlu_pro": 0.6164,
"buried": 0.7414,
"unknowable": 0.4128,
"unknowable_control": 0.9795,
"aqua_rat": 0.6181,
"cosmos_qa": 0.9,
"esci": 0.5933,
"glaive_fc": 0.9933,
"gsm8k": 0.755,
"hellaswag": 0.9536,
"newyorker": 0.7415,
"openbookqa": 0.8933,
"ragtruth": 0.7933,
"ruletaker": 0.94,
"sgd": 0.9733,
"swag": 0.8267,
"toolace": 0.9533,
"when2call": 0.9467,
"winogrande": 0.92
},
"unknowable": {
"n": 109,
"mean_max_p": 0.5008,
"share_at_0_9": 0.0183,
"control_acc": 0.9795,
"control_mean_max_p": 0.9638,
"control_share_at_0_9": 0.8699
}
},
"training_config": {
"model": "Qwen/Qwen3.5-9B",
"init": null,
"lora_r": 16,
"lora_alpha": 32.0,
"head_dim": 256,
"eval_batch_size": 16,
"eval_think_limit": 256,
"seed": 0,
"max_think": 2560,
"temperature": 1.0,
"gen_batch_size": 64,
"pad_multiple": 256,
"num_workers": 2,
"compile": true,
"stage": "sft",
"run_dir": "runs/sft",
"data_dir": "data",
"resume": null,
"lr": 5e-05,
"head_lr": 0.0001,
"warmup": 20,
"sft_epochs": 1,
"cispo_epochs": 4,
"stop_step": 402,
"batch_size": 8,
"grad_clip": 1.0,
"max_len": 4096,
"max_seq_len": 8192,
"train_limit": 0,
"dev_limit": 0,
"eval_every": 2000,
"eval_think": true,
"log_every": 10,
"gradient_checkpointing": true,
"prompts_per_step": 8,
"group": 8,
"eps_high": 0.2,
"rollout_ce_weight": 0.5,
"anchor_ce_weight": 0.5,
"anchor_batch": 8,
"anchor_limit": 0,
"logprob_chunk": 128,
"micro_groups": 2,
"length_hinge": 2048,
"length_hinge_end": 1024,
"length_penalty_cap": 0.1,
"ckpt_minutes": 45.0,
"log_rollouts": 3,
"log_chain_chars": 1200,
"sft_think_frac": 0.15,
"chain_cache": "data/chains",
"bucket": true,
"chain_max_len": 1536,
"unclosed_reward": 1.0,
"loop_penalty": 0.0,
"greedy_rollouts": 0,
"chain_lm_weight": 0.5,
"sft_truncate_frac": 0.2
}
}