nm-autobot commited on
Commit
1f3ab0c
·
verified ·
1 Parent(s): 0c51464

Upload folder using huggingface_hub

Browse files
Files changed (4) hide show
  1. config.json +8 -23
  2. generation_config.json +1 -1
  3. model.safetensors +1 -1
  4. recipe.yaml +7 -27
config.json CHANGED
@@ -3,58 +3,43 @@
3
  "LlamaForCausalLM"
4
  ],
5
  "attention_bias": false,
6
- "attention_dropout": 0.0,
7
  "bos_token_id": 1,
8
- "dtype": "bfloat16",
9
  "eos_token_id": 2,
10
- "head_dim": 64,
11
  "hidden_act": "silu",
12
  "hidden_size": 2048,
13
  "initializer_range": 0.02,
14
  "intermediate_size": 5632,
15
  "max_position_embeddings": 2048,
16
- "mlp_bias": false,
17
  "model_type": "llama",
18
  "num_attention_heads": 32,
19
  "num_hidden_layers": 22,
20
  "num_key_value_heads": 4,
21
- "pad_token_id": null,
22
  "pretraining_tp": 1,
23
  "quantization_config": {
24
  "config_groups": {
25
  "group_0": {
26
  "format": "int-quantized",
27
  "input_activations": {
28
- "actorder": null,
29
- "block_structure": null,
30
  "dynamic": false,
31
  "group_size": null,
32
  "num_bits": 8,
33
  "observer": "static_minmax",
34
- "observer_kwargs": {},
35
- "scale_dtype": null,
36
  "strategy": "tensor",
37
  "symmetric": false,
38
- "type": "int",
39
- "zp_dtype": "torch.int8"
40
  },
41
  "output_activations": null,
42
  "targets": [
43
  "Linear"
44
  ],
45
  "weights": {
46
- "actorder": null,
47
- "block_structure": null,
48
  "dynamic": false,
49
  "group_size": null,
50
  "num_bits": 8,
51
  "observer": "memoryless_minmax",
52
- "observer_kwargs": {},
53
- "scale_dtype": null,
54
  "strategy": "channel",
55
  "symmetric": true,
56
- "type": "int",
57
- "zp_dtype": null
58
  }
59
  }
60
  },
@@ -64,19 +49,19 @@
64
  "lm_head"
65
  ],
66
  "kv_cache_scheme": null,
 
67
  "quant_method": "compressed-tensors",
68
  "quantization_status": "compressed",
69
  "sparsity_config": {},
70
  "transform_config": {},
71
- "version": "0.17.1"
72
  },
73
  "rms_norm_eps": 1e-05,
74
- "rope_parameters": {
75
- "rope_theta": 10000.0,
76
- "rope_type": "default"
77
- },
78
  "tie_word_embeddings": false,
79
- "transformers_version": "5.10.1",
 
80
  "use_cache": true,
81
  "vocab_size": 32000
82
  }
 
3
  "LlamaForCausalLM"
4
  ],
5
  "attention_bias": false,
 
6
  "bos_token_id": 1,
 
7
  "eos_token_id": 2,
 
8
  "hidden_act": "silu",
9
  "hidden_size": 2048,
10
  "initializer_range": 0.02,
11
  "intermediate_size": 5632,
12
  "max_position_embeddings": 2048,
 
13
  "model_type": "llama",
14
  "num_attention_heads": 32,
15
  "num_hidden_layers": 22,
16
  "num_key_value_heads": 4,
 
17
  "pretraining_tp": 1,
18
  "quantization_config": {
19
  "config_groups": {
20
  "group_0": {
21
  "format": "int-quantized",
22
  "input_activations": {
 
 
23
  "dynamic": false,
24
  "group_size": null,
25
  "num_bits": 8,
26
  "observer": "static_minmax",
 
 
27
  "strategy": "tensor",
28
  "symmetric": false,
29
+ "type": "int"
 
30
  },
31
  "output_activations": null,
32
  "targets": [
33
  "Linear"
34
  ],
35
  "weights": {
 
 
36
  "dynamic": false,
37
  "group_size": null,
38
  "num_bits": 8,
39
  "observer": "memoryless_minmax",
 
 
40
  "strategy": "channel",
41
  "symmetric": true,
42
+ "type": "int"
 
43
  }
44
  }
45
  },
 
49
  "lm_head"
50
  ],
51
  "kv_cache_scheme": null,
52
+ "moe_sparsity_config": null,
53
  "quant_method": "compressed-tensors",
54
  "quantization_status": "compressed",
55
  "sparsity_config": {},
56
  "transform_config": {},
57
+ "version": "0.20.2a20261028"
58
  },
59
  "rms_norm_eps": 1e-05,
60
+ "rope_scaling": null,
61
+ "rope_theta": 10000.0,
 
 
62
  "tie_word_embeddings": false,
63
+ "torch_dtype": "bfloat16",
64
+ "transformers_version": "4.35.0",
65
  "use_cache": true,
66
  "vocab_size": 32000
67
  }
generation_config.json CHANGED
@@ -3,5 +3,5 @@
3
  "eos_token_id": 2,
4
  "max_length": 2048,
5
  "pad_token_id": 0,
6
- "transformers_version": "5.10.1"
7
  }
 
3
  "eos_token_id": 2,
4
  "max_length": 2048,
5
  "pad_token_id": 0,
6
+ "transformers_version": "5.21.0"
7
  }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9676dd9af2e410377f95aba604e15abc4ec9e11cf3af7074ad386bd8070e6794
3
  size 1232076790
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d6c17c4fdfa184465d9988c5106f8b757757c999a3dda273a6ac362ecd43c210
3
  size 1232076790
recipe.yaml CHANGED
@@ -1,6 +1,7 @@
1
  quant_stage:
2
  quant_modifiers:
3
  SmoothQuantModifier:
 
4
  smoothing_strength: 0.8
5
  mappings:
6
  - !!python/tuple
@@ -15,37 +16,16 @@ quant_stage:
15
  config_groups:
16
  group_0:
17
  targets: [Linear]
18
- weights:
19
- num_bits: 8
20
- type: int
21
- symmetric: true
22
- group_size: null
23
- strategy: channel
24
- block_structure: null
25
- dynamic: false
26
- actorder: null
27
- scale_dtype: null
28
- zp_dtype: null
29
- observer: memoryless_minmax
30
- observer_kwargs: {}
31
- input_activations:
32
- num_bits: 8
33
- type: int
34
- symmetric: false
35
- group_size: null
36
- strategy: tensor
37
- block_structure: null
38
- dynamic: false
39
- actorder: null
40
- scale_dtype: null
41
- zp_dtype: torch.int8
42
- observer: static_minmax
43
- observer_kwargs: {}
44
  output_activations: null
45
  format: null
46
  targets: [Linear]
47
  ignore: [lm_head]
48
  bypass_divisibility_checks: false
 
49
  block_size: 128
50
  dampening_frac: 0.01
51
- offload_hessians: false
 
1
  quant_stage:
2
  quant_modifiers:
3
  SmoothQuantModifier:
4
+ requires_calibration_data: true
5
  smoothing_strength: 0.8
6
  mappings:
7
  - !!python/tuple
 
16
  config_groups:
17
  group_0:
18
  targets: [Linear]
19
+ weights: {num_bits: 8, type: int, symmetric: true, group_size: null, strategy: channel,
20
+ dynamic: false, observer: memoryless_minmax}
21
+ input_activations: {num_bits: 8, type: int, symmetric: false, group_size: null,
22
+ strategy: tensor, dynamic: false, observer: static_minmax}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  output_activations: null
24
  format: null
25
  targets: [Linear]
26
  ignore: [lm_head]
27
  bypass_divisibility_checks: false
28
+ requires_calibration_data: true
29
  block_size: 128
30
  dampening_frac: 0.01
31
+ batched_quantization: auto