meta_001950.json: add user_config so chat_sft inherits the run's LRs
Browse files
base_checkpoints/d24_decoderstack/meta_001950.json
CHANGED
|
@@ -13,7 +13,7 @@
|
|
| 13 |
"device_batch_size": 32,
|
| 14 |
"max_seq_len": 2048,
|
| 15 |
"total_batch_size": 1048576,
|
| 16 |
-
"note": "DecoderStack-medium (d24) capture converted for nanochat. This meta was built after the fact from the run log (full_d24_w8.log); DecoderStack's capture writes only {step, code, weights} and does not emit a meta.json of its own. model_config is the nanochat GPTConfig equivalent of StackConfig -- the two architectures are the same model (both count 1,384,122,122 parameters). Sufficient for load/eval and for SFT (chat_sft builds a fresh optimizer and only optionally warm-starts it). NOT sufficient for pre-training resume: there is no dataloader_state_dict, loop_state, or optimizer state here, and DecoderStack's pre-tokenized binary loader has no state nanochat's tokenizing loader could consume -- the data order could not be continued regardless. device_batch_size/max_seq_len are the nanochat-shaped restatement of micro_batch_tokens=65,536 (= 32 x 2048) -- DecoderStack trains varlen, so its micro-batch is a token count, not a sequence count. TOKENIZER: these weights use the 32k vocab shipped in the ChrisMcCormick/climbmix_32k_8_170 dataset repo (mirrored under tokenizer/ in this repo). It is NOT the same vocabulary as ChrisMcCormick/nanochat-varlen-d24-2026-03-22 -- 31,474 of 32,759 ids differ. Pairing these weights with that tokenizer produces garbage.",
|
| 17 |
"stack_config": {
|
| 18 |
"n_layers": 24,
|
| 19 |
"d_model": 1536,
|
|
@@ -83,5 +83,25 @@
|
|
| 83 |
"smear_lr": 0.2,
|
| 84 |
"adamw_eps": 1e-10
|
| 85 |
},
|
| 86 |
-
"val_bpb_note": "not measured at this step (val_loss_every=250); bracketed by step 1750 = 0.815902 and step 2000 = 0.809199. This is the last step before the LR/momentum cooldown begins."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
}
|
|
|
|
| 13 |
"device_batch_size": 32,
|
| 14 |
"max_seq_len": 2048,
|
| 15 |
"total_batch_size": 1048576,
|
| 16 |
+
"note": "DecoderStack-medium (d24) capture converted for nanochat. This meta was built after the fact from the run log (full_d24_w8.log); DecoderStack's capture writes only {step, code, weights} and does not emit a meta.json of its own. model_config is the nanochat GPTConfig equivalent of StackConfig -- the two architectures are the same model (both count 1,384,122,122 parameters). Sufficient for load/eval and for SFT (chat_sft builds a fresh optimizer and only optionally warm-starts it). NOT sufficient for pre-training resume: there is no dataloader_state_dict, loop_state, or optimizer state here, and DecoderStack's pre-tokenized binary loader has no state nanochat's tokenizing loader could consume -- the data order could not be continued regardless. device_batch_size/max_seq_len are the nanochat-shaped restatement of micro_batch_tokens=65,536 (= 32 x 2048) -- DecoderStack trains varlen, so its micro-batch is a token count, not a sequence count. TOKENIZER: these weights use the 32k vocab shipped in the ChrisMcCormick/climbmix_32k_8_170 dataset repo (mirrored under tokenizer/ in this repo). It is NOT the same vocabulary as ChrisMcCormick/nanochat-varlen-d24-2026-03-22 -- 31,474 of 32,759 ids differ. Pairing these weights with that tokenizer produces garbage. USER_CONFIG added 2026-08-01 (make_user_config.py): chat_sft reads the pretraining LRs from meta['user_config'], so without this block it would have silently inherited its own fallback unembedding_lr=0.004 instead of this run's 0.008. Values are copied from the `schedules` / `stack_config` blocks already in this file; base_train.py argparse fields this capture gives no evidence for are omitted rather than guessed. weight_decay=0.28 is the pre-scale argument that yields the recorded muon_weight_decay_peak of 0.059738.",
|
| 17 |
"stack_config": {
|
| 18 |
"n_layers": 24,
|
| 19 |
"d_model": 1536,
|
|
|
|
| 83 |
"smear_lr": 0.2,
|
| 84 |
"adamw_eps": 1e-10
|
| 85 |
},
|
| 86 |
+
"val_bpb_note": "not measured at this step (val_loss_every=250); bracketed by step 1750 = 0.815902 and step 2000 = 0.809199. This is the last step before the LR/momentum cooldown begins.",
|
| 87 |
+
"user_config": {
|
| 88 |
+
"depth": 24,
|
| 89 |
+
"aspect_ratio": 64,
|
| 90 |
+
"head_dim": 128,
|
| 91 |
+
"max_seq_len": 2048,
|
| 92 |
+
"window_pattern": "SSSL",
|
| 93 |
+
"num_iterations": 5568,
|
| 94 |
+
"device_batch_size": 32,
|
| 95 |
+
"total_batch_size": 1048576,
|
| 96 |
+
"embedding_lr": 0.3,
|
| 97 |
+
"unembedding_lr": 0.008,
|
| 98 |
+
"matrix_lr": 0.02,
|
| 99 |
+
"scalar_lr": 0.5,
|
| 100 |
+
"weight_decay": 0.28,
|
| 101 |
+
"warmup_steps": 40,
|
| 102 |
+
"warmdown_ratio": 0.65,
|
| 103 |
+
"final_lr_frac": 0.05,
|
| 104 |
+
"model_tag": "d24_decoderstack",
|
| 105 |
+
"run": "2026-08-01_023219-d24"
|
| 106 |
+
}
|
| 107 |
}
|