ChrisMcCormick commited on
Commit
1bfa5e5
·
verified ·
1 Parent(s): 7b1ab30

meta_001950.json: add user_config so chat_sft inherits the run's LRs

Browse files
base_checkpoints/d24_decoderstack/meta_001950.json CHANGED
@@ -13,7 +13,7 @@
13
  "device_batch_size": 32,
14
  "max_seq_len": 2048,
15
  "total_batch_size": 1048576,
16
- "note": "DecoderStack-medium (d24) capture converted for nanochat. This meta was built after the fact from the run log (full_d24_w8.log); DecoderStack's capture writes only {step, code, weights} and does not emit a meta.json of its own. model_config is the nanochat GPTConfig equivalent of StackConfig -- the two architectures are the same model (both count 1,384,122,122 parameters). Sufficient for load/eval and for SFT (chat_sft builds a fresh optimizer and only optionally warm-starts it). NOT sufficient for pre-training resume: there is no dataloader_state_dict, loop_state, or optimizer state here, and DecoderStack's pre-tokenized binary loader has no state nanochat's tokenizing loader could consume -- the data order could not be continued regardless. device_batch_size/max_seq_len are the nanochat-shaped restatement of micro_batch_tokens=65,536 (= 32 x 2048) -- DecoderStack trains varlen, so its micro-batch is a token count, not a sequence count. TOKENIZER: these weights use the 32k vocab shipped in the ChrisMcCormick/climbmix_32k_8_170 dataset repo (mirrored under tokenizer/ in this repo). It is NOT the same vocabulary as ChrisMcCormick/nanochat-varlen-d24-2026-03-22 -- 31,474 of 32,759 ids differ. Pairing these weights with that tokenizer produces garbage.",
17
  "stack_config": {
18
  "n_layers": 24,
19
  "d_model": 1536,
@@ -83,5 +83,25 @@
83
  "smear_lr": 0.2,
84
  "adamw_eps": 1e-10
85
  },
86
- "val_bpb_note": "not measured at this step (val_loss_every=250); bracketed by step 1750 = 0.815902 and step 2000 = 0.809199. This is the last step before the LR/momentum cooldown begins."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  }
 
13
  "device_batch_size": 32,
14
  "max_seq_len": 2048,
15
  "total_batch_size": 1048576,
16
+ "note": "DecoderStack-medium (d24) capture converted for nanochat. This meta was built after the fact from the run log (full_d24_w8.log); DecoderStack's capture writes only {step, code, weights} and does not emit a meta.json of its own. model_config is the nanochat GPTConfig equivalent of StackConfig -- the two architectures are the same model (both count 1,384,122,122 parameters). Sufficient for load/eval and for SFT (chat_sft builds a fresh optimizer and only optionally warm-starts it). NOT sufficient for pre-training resume: there is no dataloader_state_dict, loop_state, or optimizer state here, and DecoderStack's pre-tokenized binary loader has no state nanochat's tokenizing loader could consume -- the data order could not be continued regardless. device_batch_size/max_seq_len are the nanochat-shaped restatement of micro_batch_tokens=65,536 (= 32 x 2048) -- DecoderStack trains varlen, so its micro-batch is a token count, not a sequence count. TOKENIZER: these weights use the 32k vocab shipped in the ChrisMcCormick/climbmix_32k_8_170 dataset repo (mirrored under tokenizer/ in this repo). It is NOT the same vocabulary as ChrisMcCormick/nanochat-varlen-d24-2026-03-22 -- 31,474 of 32,759 ids differ. Pairing these weights with that tokenizer produces garbage. USER_CONFIG added 2026-08-01 (make_user_config.py): chat_sft reads the pretraining LRs from meta['user_config'], so without this block it would have silently inherited its own fallback unembedding_lr=0.004 instead of this run's 0.008. Values are copied from the `schedules` / `stack_config` blocks already in this file; base_train.py argparse fields this capture gives no evidence for are omitted rather than guessed. weight_decay=0.28 is the pre-scale argument that yields the recorded muon_weight_decay_peak of 0.059738.",
17
  "stack_config": {
18
  "n_layers": 24,
19
  "d_model": 1536,
 
83
  "smear_lr": 0.2,
84
  "adamw_eps": 1e-10
85
  },
86
+ "val_bpb_note": "not measured at this step (val_loss_every=250); bracketed by step 1750 = 0.815902 and step 2000 = 0.809199. This is the last step before the LR/momentum cooldown begins.",
87
+ "user_config": {
88
+ "depth": 24,
89
+ "aspect_ratio": 64,
90
+ "head_dim": 128,
91
+ "max_seq_len": 2048,
92
+ "window_pattern": "SSSL",
93
+ "num_iterations": 5568,
94
+ "device_batch_size": 32,
95
+ "total_batch_size": 1048576,
96
+ "embedding_lr": 0.3,
97
+ "unembedding_lr": 0.008,
98
+ "matrix_lr": 0.02,
99
+ "scalar_lr": 0.5,
100
+ "weight_decay": 0.28,
101
+ "warmup_steps": 40,
102
+ "warmdown_ratio": 0.65,
103
+ "final_lr_frac": 0.05,
104
+ "model_tag": "d24_decoderstack",
105
+ "run": "2026-08-01_023219-d24"
106
+ }
107
  }