EngRandom_1.0 / tokenizer.json
KomorAI's picture
Upload model EngRandom_1.0 + quantizations + auto README
069d8a7 verified
Raw History Blame Contribute Delete
9.15 kB
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 32,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 1,
"pad_type_id": 0,
"pad_token": "[PAD]"
},
"added_tokens": [
{
"id": 0,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {}
},
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"[UNK]": 0,
"[PAD]": 1,
"<|endoftext|>": 2,
"hob": 3,
"twee": 4,
"dense": 5,
"run": 6,
"nuts": 7,
"fever": 8,
"fetch": 9,
"rival": 10,
"consider": 11,
"occasionally": 12,
"upchange": 13,
"mauve": 14,
"extricate": 15,
"pass": 16,
"pain": 17,
"dilute": 18,
"prune": 19,
"stake": 20,
"impregnate": 21,
"oral": 22,
"tree": 23,
"holster": 24,
"donor": 25,
"ample": 26,
"petition": 27,
"mooch": 28,
"exciting": 29,
"sob": 30,
"scan": 31,
"deaden": 32,
"runic": 33,
"wily": 34,
"lecture": 35,
"taboo": 36,
"breakfast": 37,
"evoke": 38,
"saver": 39,
"teeth": 40,
"void": 41,
"jolt": 42,
"grief": 43,
"ring": 44,
"black": 45,
"livid": 46,
"golem": 47,
"nearly": 48,
"imagine": 49,
"baby": 50,
"weary": 51,
"mural": 52,
"mushy": 53,
"rank": 54,
"mod": 55,
"album": 56,
"lion": 57,
"zebra": 58,
"own": 59,
"pied": 60,
"urinate": 61,
"lambast": 62,
"ambush": 63,
"mothball": 64,
"corner": 65,
"rash": 66,
"two": 67,
"faster": 68,
"mix": 69,
"face": 70,
"ticket": 71,
"wean": 72,
"successful": 73,
"regard": 74,
"belly": 75,
"feel": 76,
"disrobe": 77,
"ruled": 78,
"dread": 79,
"fine": 80,
"muddy": 81,
"cat": 82,
"salon": 83,
"once": 84,
"plow": 85,
"macro": 86,
"rasp": 87,
"smoky": 88,
"shiny": 89,
"live": 90,
"view": 91,
"plan": 92,
"prize": 93,
"coerce": 94,
"grass": 95,
"mighty": 96,
"lift": 97,
"take": 98,
"entire": 99,
"rear": 100,
"smile": 101,
"round": 102,
"hasty": 103,
"comfort": 104,
"tiger": 105,
"plump": 106,
"lane": 107,
"enjoy": 108,
"ruminate": 109,
"glad": 110,
"damn": 111,
"rife": 112,
"indicate": 113,
"bleat": 114,
"remount": 115,
"clump": 116,
"clock": 117,
"usually": 118,
"force": 119,
"husky": 120,
"mummy": 121,
"growl": 122,
"pony": 123,
"jolly": 124,
"line": 125,
"fantasize": 126,
"knowledge": 127,
"spine": 128,
"achieve": 129,
"inseminate": 130,
"tow": 131,
"gummy": 132,
"incommode": 133,
"kill": 134,
"valet": 135,
"rosin": 136,
"hazy": 137,
"bared": 138,
"summer": 139,
"sewn": 140,
"girl": 141,
"search": 142,
"freedom": 143,
"reason": 144,
"bet": 145,
"testy": 146,
"shed": 147,
"tidy": 148,
"tea": 149,
"vague": 150,
"skulk": 151,
"world": 152,
"retro": 153,
"some": 154,
"dead": 155,
"snow": 156,
"reset": 157,
"structure": 158,
"dab": 159,
"carbonise": 160,
"chart": 161,
"wild": 162,
"fighting": 163,
"loud": 164,
"gent": 165,
"cross": 166,
"kick": 167,
"rocky": 168,
"blag": 169,
"tubby": 170,
"crawl": 171,
"have": 172,
"rosy": 173,
"dial": 174,
"blear": 175,
"strap": 176,
"make": 177,
"mock": 178,
"habit": 179,
"lunge": 180,
"secede": 181,
"ego": 182,
"spumy": 183,
"warm": 184,
"raise": 185,
"bar": 186,
"pace": 187,
"rubber": 188,
"taken": 189,
"paper": 190,
"conquer": 191,
"three": 192,
"lefty": 193,
"right": 194,
"tub": 195,
"carefully": 196,
"nick": 197,
"darn": 198,
"stonewall": 199,
"nice": 200,
"cadet": 201,
"mince": 202,
"bated": 203,
"shut": 204,
"bumble": 205,
"sing": 206,
"job": 207,
"picture": 208,
"redeploy": 209,
"cache": 210,
"noted": 211,
"raw": 212,
"decorate": 213,
"loamy": 214,
"seal": 215,
"lemur": 216,
"below": 217,
"fun": 218,
"hoagy": 219,
"peal": 220,
"quick": 221,
"ready": 222,
"gate": 223,
"string": 224,
"slate": 225,
"nanna": 226,
"row": 227,
"filch": 228,
"choice": 229,
"struggle": 230,
"purloin": 231,
"space": 232,
"pat": 233,
"gesture": 234,
"lope": 235,
"hops": 236,
"recall": 237,
"abet": 238,
"pricy": 239,
"mimer": 240,
"arm": 241,
"fated": 242,
"promo": 243,
"empty": 244,
"antic": 245,
"arrow": 246,
"four": 247,
"awash": 248,
"natural": 249,
"times": 250,
"tape": 251,
"beat": 252,
"arms": 253,
"sour": 254,
"moony": 255,
"mantle": 256,
"fashion": 257,
"showy": 258,
"shawl": 259,
"thrill": 260,
"room": 261,
"swing": 262,
"ridge": 263,
"parts": 264,
"astir": 265,
"has": 266,
"fluid": 267,
"diet": 268,
"port": 269,
"oaken": 270,
"too": 271,
"stopped": 272,
"video": 273,
"ink": 274,
"able": 275,
"frail": 276,
"favor": 277,
"catalyze": 278,
"wacky": 279,
"factor": 280,
"limp": 281,
"rib": 282,
"zany": 283,
"chuckle": 284,
"lay": 285,
"color": 286,
"discard": 287,
"replied": 288,
"summon": 289,
"fifth": 290,
"jumbo": 291,
"fabricate": 292,
"fruit": 293,
"pop": 294,
"fracture": 295,
"remix": 296,
"low": 297,
"fence": 298,
"brand": 299,
"men": 300,
"basin": 301,
"kinky": 302,
"hump": 303,
"evil": 304,
"gawky": 305,
"team": 306,
"noise": 307,
"liken": 308,
"slew": 309,
"elude": 310,
"scandalize": 311,
"tribe": 312,
"lank": 313,
"flex": 314,
"farce": 315,
"you": 316,
"timer": 317,
"shower": 318,
"sugar": 319,
"ion": 320,
"bliss": 321,
"instrument": 322,
"dowel": 323,
"nag": 324,
"redo": 325,
"quell": 326,
"comfortable": 327,
"fortify": 328,
"task": 329,
"elder": 330,
"expiate": 331,
"dip": 332,
"mangy": 333,
"plane": 334,
"lent": 335,
"till": 336,
"rejoice": 337,
"drape": 338,
"cello": 339,
"flavour": 340,
"frank": 341,
"cleft": 342,
"crisp": 343,
"tip": 344,
"halo": 345,
"television": 346,
"agonize": 347,
"wool": 348,
"muggy": 349,
"ham": 350,
"aqua": 351,
"palmy": 352,
"found": 353,
"produce": 354,
"leach": 355,
"doing": 356,
"wits": 357,
"forum": 358,
"grain": 359,
"legitimise": 360,
"spoil": 361,
"walk": 362,
"indeed": 363,
"clothes": 364,
"soak": 365,
"sixty": 366,
"using": 367,
"babble": 368,
"hail": 369,
"ordinary": 370,
"shot": 371,
"ten": 372,
"barn": 373,
"gulch": 374,
"seat": 375,
"during": 376,
"acme": 377,
"vial": 378,
"herald": 379,
"tomorrow": 380,
"fudge": 381,
"mice": 382,
"lowly": 383,
"goon": 384,
"ace": 385,
"rake": 386
},
"unk_token": "[UNK]"
}
}