Download tokenizer.json from KomorAI/EngRandom_1.0_Pro: direct link, hf CLI and curl.
- Browser
- Download file 20.1 kB
-
https://huggingface.co/KomorAI/EngRandom_1.0_Pro/resolve/main/tokenizer.json
- Command line
-
hf download hf://KomorAI/EngRandom_1.0_Pro/tokenizer.json
-
curl -L -o tokenizer.json https://huggingface.co/KomorAI/EngRandom_1.0_Pro/resolve/main/tokenizer.json
20.1 kB
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 32,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 1,
"pad_type_id": 0,
"pad_token": "[PAD]"
},
"added_tokens": [
{
"id": 0,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {}
},
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"[UNK]": 0,
"[PAD]": 1,
"<|endoftext|>": 2,
"prefigure": 3,
"follow": 4,
"metal": 5,
"rear": 6,
"flirt": 7,
"berate": 8,
"medic": 9,
"bonny": 10,
"pupil": 11,
"curvy": 12,
"roast": 13,
"empty": 14,
"package": 15,
"afire": 16,
"louse": 17,
"goes": 18,
"find": 19,
"skate": 20,
"later": 21,
"downy": 22,
"place": 23,
"seed": 24,
"diary": 25,
"while": 26,
"dishonor": 27,
"had": 28,
"combo": 29,
"predicate": 30,
"valid": 31,
"braces": 32,
"boon": 33,
"lanky": 34,
"speed": 35,
"accustom": 36,
"chunter": 37,
"husky": 38,
"pun": 39,
"due": 40,
"attempt": 41,
"dispatch": 42,
"adult": 43,
"seven": 44,
"dud": 45,
"manner": 46,
"type": 47,
"minute": 48,
"oval": 49,
"mine": 50,
"ham": 51,
"site": 52,
"volumize": 53,
"mod": 54,
"segue": 55,
"individualize": 56,
"coy": 57,
"devalue": 58,
"pert": 59,
"fab": 60,
"solve": 61,
"electric": 62,
"stem": 63,
"ended": 64,
"brown": 65,
"raspy": 66,
"panic": 67,
"mates": 68,
"basal": 69,
"scandalise": 70,
"threw": 71,
"saber": 72,
"captured": 73,
"involve": 74,
"toxic": 75,
"own": 76,
"light": 77,
"salt": 78,
"class": 79,
"drink": 80,
"cry": 81,
"misappropriate": 82,
"jawed": 83,
"stuff": 84,
"junk": 85,
"maxim": 86,
"angry": 87,
"awed": 88,
"tamp": 89,
"motion": 90,
"dropped": 91,
"stare": 92,
"balk": 93,
"numb": 94,
"which": 95,
"sin": 96,
"yawn": 97,
"glide": 98,
"stark": 99,
"crisp": 100,
"dusk": 101,
"individual": 102,
"brute": 103,
"belt": 104,
"alien": 105,
"serve": 106,
"unblock": 107,
"action": 108,
"storm": 109,
"spay": 110,
"range": 111,
"abate": 112,
"sang": 113,
"fire": 114,
"strip": 115,
"slacken": 116,
"nigh": 117,
"tense": 118,
"rain": 119,
"yam": 120,
"apartment": 121,
"straggle": 122,
"reset": 123,
"dead": 124,
"firm": 125,
"liquefy": 126,
"label": 127,
"huffy": 128,
"perk": 129,
"experience": 130,
"driving": 131,
"perfectly": 132,
"pressurise": 133,
"convert": 134,
"resound": 135,
"scare": 136,
"huge": 137,
"party": 138,
"eats": 139,
"motel": 140,
"faint": 141,
"link": 142,
"bong": 143,
"rough": 144,
"tar": 145,
"flint": 146,
"blind": 147,
"creosote": 148,
"worry": 149,
"solution": 150,
"repudiate": 151,
"owing": 152,
"lazy": 153,
"fumed": 154,
"worker": 155,
"sire": 156,
"mistake": 157,
"tent": 158,
"temperature": 159,
"century": 160,
"catch": 161,
"nick": 162,
"joint": 163,
"gabble": 164,
"avian": 165,
"leave": 166,
"wry": 167,
"polar": 168,
"ran": 169,
"hamper": 170,
"sold": 171,
"bated": 172,
"drain": 173,
"musky": 174,
"igloo": 175,
"cabin": 176,
"echo": 177,
"dice": 178,
"feat": 179,
"curl": 180,
"commute": 181,
"polymerize": 182,
"miscarry": 183,
"awry": 184,
"caught": 185,
"question": 186,
"novel": 187,
"alarm": 188,
"crimp": 189,
"rich": 190,
"grant": 191,
"block": 192,
"gawk": 193,
"tours": 194,
"main": 195,
"fly": 196,
"askew": 197,
"stray": 198,
"aura": 199,
"holey": 200,
"hall": 201,
"wispy": 202,
"somebody": 203,
"wacky": 204,
"pedal": 205,
"pink": 206,
"beach": 207,
"bushy": 208,
"excited": 209,
"fig": 210,
"rearm": 211,
"last": 212,
"mastermind": 213,
"stir": 214,
"telephone": 215,
"portray": 216,
"none": 217,
"shady": 218,
"gerrymander": 219,
"share": 220,
"mulch": 221,
"begrudge": 222,
"hick": 223,
"laser": 224,
"cause": 225,
"shaking": 226,
"kind": 227,
"cinch": 228,
"rowdy": 229,
"pouch": 230,
"brief": 231,
"palmy": 232,
"come": 233,
"funny": 234,
"plot": 235,
"clue": 236,
"hill": 237,
"physical": 238,
"gob": 239,
"spray": 240,
"convict": 241,
"briny": 242,
"lay": 243,
"product": 244,
"idol": 245,
"captain": 246,
"style": 247,
"mark": 248,
"so": 249,
"miter": 250,
"anti": 251,
"discomfit": 252,
"fellate": 253,
"ready": 254,
"resign": 255,
"fixer": 256,
"struggle": 257,
"famed": 258,
"slant": 259,
"void": 260,
"low": 261,
"deaf": 262,
"whinny": 263,
"adventure": 264,
"hull": 265,
"peak": 266,
"hurt": 267,
"leverage": 268,
"germ": 269,
"scientific": 270,
"stunt": 271,
"cage": 272,
"water": 273,
"like": 274,
"grandfather": 275,
"yield": 276,
"contest": 277,
"dyed": 278,
"flaky": 279,
"inept": 280,
"fret": 281,
"gaunt": 282,
"pug": 283,
"zany": 284,
"weak": 285,
"accurate": 286,
"map": 287,
"sorry": 288,
"grove": 289,
"petty": 290,
"thick": 291,
"pearl": 292,
"kinky": 293,
"nubby": 294,
"irony": 295,
"wag": 296,
"enjoy": 297,
"brood": 298,
"stout": 299,
"pupal": 300,
"unravel": 301,
"lemur": 302,
"browse": 303,
"rabbit": 304,
"duckie": 305,
"pet": 306,
"discontinue": 307,
"cattle": 308,
"blanket": 309,
"blood": 310,
"experiment": 311,
"lusty": 312,
"golden": 313,
"avid": 314,
"silence": 315,
"either": 316,
"fort": 317,
"object": 318,
"lent": 319,
"spots": 320,
"deputise": 321,
"self": 322,
"rest": 323,
"brave": 324,
"trade": 325,
"rummy": 326,
"apple": 327,
"sunny": 328,
"entire": 329,
"publicise": 330,
"blear": 331,
"destroy": 332,
"alone": 333,
"whim": 334,
"sand": 335,
"swarm": 336,
"record": 337,
"fable": 338,
"eased": 339,
"mission": 340,
"denote": 341,
"participate": 342,
"early": 343,
"disc": 344,
"odds": 345,
"nine": 346,
"deli": 347,
"sulky": 348,
"acid": 349,
"chug": 350,
"reproduce": 351,
"glad": 352,
"fancy": 353,
"swan": 354,
"braai": 355,
"prior": 356,
"carburize": 357,
"enlarge": 358,
"skid": 359,
"opposite": 360,
"wiry": 361,
"highest": 362,
"bum": 363,
"whack": 364,
"afoul": 365,
"bossy": 366,
"prosy": 367,
"livid": 368,
"anyone": 369,
"keyed": 370,
"wired": 371,
"tried": 372,
"wager": 373,
"flame": 374,
"wear": 375,
"aspen": 376,
"number": 377,
"boy": 378,
"formalise": 379,
"hump": 380,
"axe": 381,
"worst": 382,
"gimpy": 383,
"seldom": 384,
"quad": 385,
"habit": 386,
"weight": 387,
"male": 388,
"orbit": 389,
"that": 390,
"burn": 391,
"hazel": 392,
"snug": 393,
"study": 394,
"monopolise": 395,
"pact": 396,
"forage": 397,
"misinterpret": 398,
"bared": 399,
"twig": 400,
"quid": 401,
"mirky": 402,
"bark": 403,
"envisage": 404,
"demilitarise": 405,
"point": 406,
"steps": 407,
"disappear": 408,
"check": 409,
"cup": 410,
"wink": 411,
"strong": 412,
"dosed": 413,
"dummy": 414,
"fossilize": 415,
"complex": 416,
"snaky": 417,
"lash": 418,
"paper": 419,
"opinion": 420,
"incentivize": 421,
"more": 422,
"bruit": 423,
"broke": 424,
"word": 425,
"acute": 426,
"rectify": 427,
"partly": 428,
"saved": 429,
"summer": 430,
"stuck": 431,
"squeeze": 432,
"cut": 433,
"much": 434,
"fend": 435,
"energy": 436,
"sight": 437,
"deep": 438,
"rosy": 439,
"waxen": 440,
"cart": 441,
"accept": 442,
"lofty": 443,
"noise": 444,
"adjust": 445,
"sell": 446,
"beautify": 447,
"demineralize": 448,
"rugby": 449,
"pushy": 450,
"grass": 451,
"pedestrianise": 452,
"built": 453,
"improve": 454,
"grim": 455,
"excoriate": 456,
"overhang": 457,
"pilar": 458,
"glory": 459,
"rabid": 460,
"month": 461,
"biome": 462,
"substantiate": 463,
"quiver": 464,
"eve": 465,
"white": 466,
"bored": 467,
"depersonalize": 468,
"gross": 469,
"exasperate": 470,
"father": 471,
"stay": 472,
"sea": 473,
"misread": 474,
"basil": 475,
"wore": 476,
"know": 477,
"wrath": 478,
"desegregate": 479,
"rover": 480,
"prosecute": 481,
"hoot": 482,
"birthday": 483,
"rug": 484,
"reoffend": 485,
"fuse": 486,
"stigmatize": 487,
"tap": 488,
"rush": 489,
"flat": 490,
"wed": 491,
"moon": 492,
"prism": 493,
"trace": 494,
"mollify": 495,
"sky": 496,
"quack": 497,
"bite": 498,
"ozone": 499,
"pen": 500,
"boss": 501,
"stomp": 502,
"pin": 503,
"exempt": 504,
"corner": 505,
"regress": 506,
"spawn": 507,
"twelve": 508,
"ample": 509,
"mild": 510,
"narrate": 511,
"scope": 512,
"relationship": 513,
"television": 514,
"edgy": 515,
"outer": 516,
"restrain": 517,
"plump": 518,
"cross": 519,
"fat": 520,
"use": 521,
"reinterpret": 522,
"loco": 523,
"position": 524,
"ease": 525,
"pitch": 526,
"hazy": 527,
"displease": 528,
"mad": 529,
"wolf": 530,
"jig": 531,
"sublimate": 532,
"levy": 533,
"jazzy": 534,
"thumb": 535,
"harbor": 536,
"wrapped": 537,
"grind": 538,
"tough": 539,
"posed": 540,
"jacks": 541,
"read": 542,
"neuter": 543,
"manhandle": 544,
"hundred": 545,
"juke": 546,
"loopy": 547,
"coming": 548,
"overstate": 549,
"stomach": 550,
"oasis": 551,
"pivot": 552,
"boxy": 553,
"furry": 554,
"quiet": 555,
"halo": 556,
"scan": 557,
"mysterious": 558,
"pride": 559,
"throbbing": 560,
"gold": 561,
"editorialize": 562,
"recast": 563,
"intermarry": 564,
"behove": 565,
"chief": 566,
"remortgage": 567,
"maths": 568,
"cling": 569,
"draw": 570,
"tears": 571,
"tulip": 572,
"broad": 573,
"tot": 574,
"cuff": 575,
"sense": 576,
"breeze": 577,
"divide": 578,
"utter": 579,
"pain": 580,
"deny": 581,
"goner": 582,
"rapid": 583,
"wise": 584,
"differ": 585,
"such": 586,
"equal": 587,
"radio": 588,
"octet": 589,
"home": 590,
"design": 591,
"note": 592,
"slice": 593,
"avoid": 594,
"blond": 595,
"vitrify": 596,
"cloth": 597,
"require": 598,
"tremble": 599,
"no": 600,
"mask": 601,
"book": 602,
"recede": 603,
"chillax": 604,
"noose": 605,
"braise": 606,
"akin": 607,
"every": 608,
"roof": 609,
"realize": 610,
"spoof": 611,
"cool": 612,
"duct": 613,
"art": 614,
"characteristic": 615,
"cozy": 616,
"imperil": 617,
"credit": 618,
"unhappy": 619,
"you": 620,
"sneak": 621,
"many": 622,
"vary": 623,
"table": 624,
"welcome": 625,
"slav": 626,
"debrief": 627,
"punch": 628,
"evade": 629,
"box": 630,
"smoulder": 631,
"horse": 632,
"dried": 633,
"office": 634,
"wild": 635,
"neighbor": 636,
"verb": 637,
"rave": 638,
"introduced": 639,
"immediately": 640,
"unloosen": 641,
"dammit": 642,
"befall": 643,
"decoy": 644,
"snot": 645,
"disqualify": 646,
"wet": 647,
"wordy": 648,
"autowind": 649,
"welt": 650,
"win": 651,
"posh": 652,
"runic": 653,
"mown": 654,
"foot": 655,
"deem": 656,
"storyboard": 657,
"mogul": 658,
"subject": 659,
"harder": 660,
"anybody": 661,
"machine": 662,
"dozen": 663,
"kill": 664,
"dull": 665,
"made": 666,
"front": 667,
"consist": 668,
"never": 669,
"lag": 670,
"sworn": 671,
"zero": 672,
"work": 673,
"stern": 674,
"drool": 675,
"depute": 676,
"beta": 677,
"kiwi": 678,
"surly": 679,
"apish": 680,
"veil": 681,
"mangy": 682,
"mafia": 683,
"gift": 684,
"outrage": 685,
"bound": 686,
"managed": 687,
"police": 688,
"school": 689,
"convulse": 690,
"snuff": 691,
"ratty": 692,
"curb": 693,
"widen": 694,
"snake": 695,
"slash": 696,
"vapid": 697,
"regroup": 698,
"boozy": 699,
"mute": 700,
"dune": 701,
"alphabet": 702,
"carefully": 703,
"swish": 704,
"copper": 705,
"solar": 706,
"suppress": 707,
"scarf": 708,
"older": 709,
"state": 710,
"chant": 711,
"galumph": 712,
"fond": 713,
"some": 714,
"zipper": 715,
"giddy": 716,
"ladle": 717,
"shift": 718,
"worn": 719,
"balm": 720,
"lobby": 721,
"mint": 722,
"does": 723,
"indulge": 724,
"mud": 725,
"atrip": 726,
"magic": 727,
"hip": 728,
"ahead": 729,
"plane": 730,
"centre": 731,
"changing": 732,
"clamour": 733,
"swelter": 734,
"slick": 735,
"ashy": 736,
"term": 737,
"travel": 738,
"hush": 739,
"kneel": 740,
"distant": 741,
"irradiate": 742,
"create": 743,
"rigid": 744,
"glister": 745,
"repay": 746,
"hands": 747,
"familiar": 748,
"micro": 749,
"traumatize": 750,
"both": 751,
"review": 752,
"die": 753,
"snag": 754,
"diffract": 755,
"piano": 756,
"bullshit": 757,
"lost": 758,
"stream": 759,
"snub": 760,
"fought": 761,
"tush": 762,
"flies": 763,
"belong": 764,
"briar": 765,
"capitulate": 766,
"anger": 767,
"twin": 768,
"newt": 769,
"purloin": 770,
"disk": 771,
"gin": 772,
"except": 773,
"tingling": 774,
"muted": 775,
"apt": 776,
"list": 777,
"heard": 778,
"year": 779,
"trip": 780,
"pan": 781,
"or": 782,
"queer": 783,
"clay": 784,
"shin": 785,
"live": 786,
"floss": 787,
"sewn": 788,
"dairy": 789,
"cliff": 790,
"familiarize": 791,
"bee": 792,
"nearer": 793,
"toxin": 794,
"quite": 795,
"fold": 796,
"tenth": 797,
"pillory": 798,
"bet": 799,
"slug": 800,
"hub": 801,
"afternoon": 802,
"scowl": 803,
"group": 804,
"caulk": 805,
"valuable": 806,
"foster": 807,
"still": 808,
"decommission": 809,
"cat": 810,
"settlers": 811,
"decide": 812,
"fiber": 813,
"defecate": 814,
"sink": 815,
"edit": 816,
"climate": 817,
"job": 818,
"muse": 819,
"learn": 820,
"moved": 821,
"splay": 822,
"bucketful": 823,
"beady": 824,
"gown": 825,
"unsaddle": 826,
"bad": 827,
"hired": 828,
"scale": 829,
"capital": 830,
"series": 831,
"teem": 832,
"shorter": 833,
"deadhead": 834,
"flour": 835,
"haste": 836,
"color": 837,
"found": 838,
"miniaturise": 839,
"scent": 840,
"son": 841,
"clear": 842,
"sheep": 843,
"mite": 844,
"magnetize": 845,
"sure": 846,
"related": 847,
"viola": 848,
"shaky": 849,
"waste": 850,
"skill": 851,
"gain": 852,
"research": 853,
"pedestrianize": 854,
"parts": 855,
"kids": 856,
"worse": 857,
"select": 858,
"tip": 859,
"tipsy": 860,
"swag": 861,
"photocopy": 862,
"eclipse": 863,
"raw": 864,
"abase": 865,
"tact": 866,
"far": 867,
"deal": 868,
"liquid": 869,
"augment": 870,
"union": 871,
"supersized": 872,
"heavy": 873,
"end": 874,
"near": 875,
"orca": 876,
"stiff": 877,
"nationalize": 878,
"peddle": 879,
"busk": 880,
"annex": 881,
"pine": 882,
"man": 883,
"tart": 884,
"swing": 885,
"twiddle": 886,
"elect": 887,
"advice": 888,
"ban": 889,
"he": 890,
"baggy": 891,
"cast": 892,
"grit": 893,
"attic": 894,
"brim": 895,
"pit": 896,
"suborn": 897,
"freedom": 898,
"misbehave": 899,
"lying": 900,
"pelt": 901,
"outrun": 902,
"clock": 903,
"maltreat": 904,
"rue": 905,
"bundle": 906,
"sudsy": 907,
"crazy": 908,
"plus": 909,
"itch": 910,
"wrong": 911,
"ego": 912,
"riser": 913,
"runty": 914,
"prevent": 915,
"square": 916,
"college": 917,
"lossy": 918
},
"unk_token": "[UNK]"
}
}
{
"version": "1.0",
"truncation": {
"direction": "Right",
"max_length": 32,
"strategy": "LongestFirst",
"stride": 0
},
"padding": {
"strategy": "BatchLongest",
"direction": "Right",
"pad_to_multiple_of": null,
"pad_id": 1,
"pad_type_id": 0,
"pad_token": "[PAD]"
},
"added_tokens": [
{
"id": 0,
"content": "[UNK]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "[PAD]",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {}
},
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"[UNK]": 0,
"[PAD]": 1,
"<|endoftext|>": 2,
"prefigure": 3,
"follow": 4,
"metal": 5,
"rear": 6,
"flirt": 7,
"berate": 8,
"medic": 9,
"bonny": 10,
"pupil": 11,
"curvy": 12,
"roast": 13,
"empty": 14,
"package": 15,
"afire": 16,
"louse": 17,
"goes": 18,
"find": 19,
"skate": 20,