Download tokenizer.json from KomorAI/EngRandom_1.0: direct link, hf CLI and curl.
- Browser
- Download file 9.15 kB
-
https://huggingface.co/KomorAI/EngRandom_1.0/resolve/main/tokenizer.json
- Command line
-
hf download hf://KomorAI/EngRandom_1.0/tokenizer.json
-
curl -L -o tokenizer.json https://huggingface.co/KomorAI/EngRandom_1.0/resolve/main/tokenizer.json
9.15 kB
| { | |
| "version": "1.0", | |
| "truncation": { | |
| "direction": "Right", | |
| "max_length": 32, | |
| "strategy": "LongestFirst", | |
| "stride": 0 | |
| }, | |
| "padding": { | |
| "strategy": "BatchLongest", | |
| "direction": "Right", | |
| "pad_to_multiple_of": null, | |
| "pad_id": 1, | |
| "pad_type_id": 0, | |
| "pad_token": "[PAD]" | |
| }, | |
| "added_tokens": [ | |
| { | |
| "id": 0, | |
| "content": "[UNK]", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 1, | |
| "content": "[PAD]", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| }, | |
| { | |
| "id": 2, | |
| "content": "<|endoftext|>", | |
| "single_word": false, | |
| "lstrip": false, | |
| "rstrip": false, | |
| "normalized": false, | |
| "special": true | |
| } | |
| ], | |
| "normalizer": null, | |
| "pre_tokenizer": { | |
| "type": "Whitespace" | |
| }, | |
| "post_processor": { | |
| "type": "TemplateProcessing", | |
| "single": [ | |
| { | |
| "Sequence": { | |
| "id": "A", | |
| "type_id": 0 | |
| } | |
| } | |
| ], | |
| "pair": [ | |
| { | |
| "Sequence": { | |
| "id": "A", | |
| "type_id": 0 | |
| } | |
| }, | |
| { | |
| "Sequence": { | |
| "id": "B", | |
| "type_id": 1 | |
| } | |
| } | |
| ], | |
| "special_tokens": {} | |
| }, | |
| "decoder": null, | |
| "model": { | |
| "type": "WordLevel", | |
| "vocab": { | |
| "[UNK]": 0, | |
| "[PAD]": 1, | |
| "<|endoftext|>": 2, | |
| "hob": 3, | |
| "twee": 4, | |
| "dense": 5, | |
| "run": 6, | |
| "nuts": 7, | |
| "fever": 8, | |
| "fetch": 9, | |
| "rival": 10, | |
| "consider": 11, | |
| "occasionally": 12, | |
| "upchange": 13, | |
| "mauve": 14, | |
| "extricate": 15, | |
| "pass": 16, | |
| "pain": 17, | |
| "dilute": 18, | |
| "prune": 19, | |
| "stake": 20, | |
| "impregnate": 21, | |
| "oral": 22, | |
| "tree": 23, | |
| "holster": 24, | |
| "donor": 25, | |
| "ample": 26, | |
| "petition": 27, | |
| "mooch": 28, | |
| "exciting": 29, | |
| "sob": 30, | |
| "scan": 31, | |
| "deaden": 32, | |
| "runic": 33, | |
| "wily": 34, | |
| "lecture": 35, | |
| "taboo": 36, | |
| "breakfast": 37, | |
| "evoke": 38, | |
| "saver": 39, | |
| "teeth": 40, | |
| "void": 41, | |
| "jolt": 42, | |
| "grief": 43, | |
| "ring": 44, | |
| "black": 45, | |
| "livid": 46, | |
| "golem": 47, | |
| "nearly": 48, | |
| "imagine": 49, | |
| "baby": 50, | |
| "weary": 51, | |
| "mural": 52, | |
| "mushy": 53, | |
| "rank": 54, | |
| "mod": 55, | |
| "album": 56, | |
| "lion": 57, | |
| "zebra": 58, | |
| "own": 59, | |
| "pied": 60, | |
| "urinate": 61, | |
| "lambast": 62, | |
| "ambush": 63, | |
| "mothball": 64, | |
| "corner": 65, | |
| "rash": 66, | |
| "two": 67, | |
| "faster": 68, | |
| "mix": 69, | |
| "face": 70, | |
| "ticket": 71, | |
| "wean": 72, | |
| "successful": 73, | |
| "regard": 74, | |
| "belly": 75, | |
| "feel": 76, | |
| "disrobe": 77, | |
| "ruled": 78, | |
| "dread": 79, | |
| "fine": 80, | |
| "muddy": 81, | |
| "cat": 82, | |
| "salon": 83, | |
| "once": 84, | |
| "plow": 85, | |
| "macro": 86, | |
| "rasp": 87, | |
| "smoky": 88, | |
| "shiny": 89, | |
| "live": 90, | |
| "view": 91, | |
| "plan": 92, | |
| "prize": 93, | |
| "coerce": 94, | |
| "grass": 95, | |
| "mighty": 96, | |
| "lift": 97, | |
| "take": 98, | |
| "entire": 99, | |
| "rear": 100, | |
| "smile": 101, | |
| "round": 102, | |
| "hasty": 103, | |
| "comfort": 104, | |
| "tiger": 105, | |
| "plump": 106, | |
| "lane": 107, | |
| "enjoy": 108, | |
| "ruminate": 109, | |
| "glad": 110, | |
| "damn": 111, | |
| "rife": 112, | |
| "indicate": 113, | |
| "bleat": 114, | |
| "remount": 115, | |
| "clump": 116, | |
| "clock": 117, | |
| "usually": 118, | |
| "force": 119, | |
| "husky": 120, | |
| "mummy": 121, | |
| "growl": 122, | |
| "pony": 123, | |
| "jolly": 124, | |
| "line": 125, | |
| "fantasize": 126, | |
| "knowledge": 127, | |
| "spine": 128, | |
| "achieve": 129, | |
| "inseminate": 130, | |
| "tow": 131, | |
| "gummy": 132, | |
| "incommode": 133, | |
| "kill": 134, | |
| "valet": 135, | |
| "rosin": 136, | |
| "hazy": 137, | |
| "bared": 138, | |
| "summer": 139, | |
| "sewn": 140, | |
| "girl": 141, | |
| "search": 142, | |
| "freedom": 143, | |
| "reason": 144, | |
| "bet": 145, | |
| "testy": 146, | |
| "shed": 147, | |
| "tidy": 148, | |
| "tea": 149, | |
| "vague": 150, | |
| "skulk": 151, | |
| "world": 152, | |
| "retro": 153, | |
| "some": 154, | |
| "dead": 155, | |
| "snow": 156, | |
| "reset": 157, | |
| "structure": 158, | |
| "dab": 159, | |
| "carbonise": 160, | |
| "chart": 161, | |
| "wild": 162, | |
| "fighting": 163, | |
| "loud": 164, | |
| "gent": 165, | |
| "cross": 166, | |
| "kick": 167, | |
| "rocky": 168, | |
| "blag": 169, | |
| "tubby": 170, | |
| "crawl": 171, | |
| "have": 172, | |
| "rosy": 173, | |
| "dial": 174, | |
| "blear": 175, | |
| "strap": 176, | |
| "make": 177, | |
| "mock": 178, | |
| "habit": 179, | |
| "lunge": 180, | |
| "secede": 181, | |
| "ego": 182, | |
| "spumy": 183, | |
| "warm": 184, | |
| "raise": 185, | |
| "bar": 186, | |
| "pace": 187, | |
| "rubber": 188, | |
| "taken": 189, | |
| "paper": 190, | |
| "conquer": 191, | |
| "three": 192, | |
| "lefty": 193, | |
| "right": 194, | |
| "tub": 195, | |
| "carefully": 196, | |
| "nick": 197, | |
| "darn": 198, | |
| "stonewall": 199, | |
| "nice": 200, | |
| "cadet": 201, | |
| "mince": 202, | |
| "bated": 203, | |
| "shut": 204, | |
| "bumble": 205, | |
| "sing": 206, | |
| "job": 207, | |
| "picture": 208, | |
| "redeploy": 209, | |
| "cache": 210, | |
| "noted": 211, | |
| "raw": 212, | |
| "decorate": 213, | |
| "loamy": 214, | |
| "seal": 215, | |
| "lemur": 216, | |
| "below": 217, | |
| "fun": 218, | |
| "hoagy": 219, | |
| "peal": 220, | |
| "quick": 221, | |
| "ready": 222, | |
| "gate": 223, | |
| "string": 224, | |
| "slate": 225, | |
| "nanna": 226, | |
| "row": 227, | |
| "filch": 228, | |
| "choice": 229, | |
| "struggle": 230, | |
| "purloin": 231, | |
| "space": 232, | |
| "pat": 233, | |
| "gesture": 234, | |
| "lope": 235, | |
| "hops": 236, | |
| "recall": 237, | |
| "abet": 238, | |
| "pricy": 239, | |
| "mimer": 240, | |
| "arm": 241, | |
| "fated": 242, | |
| "promo": 243, | |
| "empty": 244, | |
| "antic": 245, | |
| "arrow": 246, | |
| "four": 247, | |
| "awash": 248, | |
| "natural": 249, | |
| "times": 250, | |
| "tape": 251, | |
| "beat": 252, | |
| "arms": 253, | |
| "sour": 254, | |
| "moony": 255, | |
| "mantle": 256, | |
| "fashion": 257, | |
| "showy": 258, | |
| "shawl": 259, | |
| "thrill": 260, | |
| "room": 261, | |
| "swing": 262, | |
| "ridge": 263, | |
| "parts": 264, | |
| "astir": 265, | |
| "has": 266, | |
| "fluid": 267, | |
| "diet": 268, | |
| "port": 269, | |
| "oaken": 270, | |
| "too": 271, | |
| "stopped": 272, | |
| "video": 273, | |
| "ink": 274, | |
| "able": 275, | |
| "frail": 276, | |
| "favor": 277, | |
| "catalyze": 278, | |
| "wacky": 279, | |
| "factor": 280, | |
| "limp": 281, | |
| "rib": 282, | |
| "zany": 283, | |
| "chuckle": 284, | |
| "lay": 285, | |
| "color": 286, | |
| "discard": 287, | |
| "replied": 288, | |
| "summon": 289, | |
| "fifth": 290, | |
| "jumbo": 291, | |
| "fabricate": 292, | |
| "fruit": 293, | |
| "pop": 294, | |
| "fracture": 295, | |
| "remix": 296, | |
| "low": 297, | |
| "fence": 298, | |
| "brand": 299, | |
| "men": 300, | |
| "basin": 301, | |
| "kinky": 302, | |
| "hump": 303, | |
| "evil": 304, | |
| "gawky": 305, | |
| "team": 306, | |
| "noise": 307, | |
| "liken": 308, | |
| "slew": 309, | |
| "elude": 310, | |
| "scandalize": 311, | |
| "tribe": 312, | |
| "lank": 313, | |
| "flex": 314, | |
| "farce": 315, | |
| "you": 316, | |
| "timer": 317, | |
| "shower": 318, | |
| "sugar": 319, | |
| "ion": 320, | |
| "bliss": 321, | |
| "instrument": 322, | |
| "dowel": 323, | |
| "nag": 324, | |
| "redo": 325, | |
| "quell": 326, | |
| "comfortable": 327, | |
| "fortify": 328, | |
| "task": 329, | |
| "elder": 330, | |
| "expiate": 331, | |
| "dip": 332, | |
| "mangy": 333, | |
| "plane": 334, | |
| "lent": 335, | |
| "till": 336, | |
| "rejoice": 337, | |
| "drape": 338, | |
| "cello": 339, | |
| "flavour": 340, | |
| "frank": 341, | |
| "cleft": 342, | |
| "crisp": 343, | |
| "tip": 344, | |
| "halo": 345, | |
| "television": 346, | |
| "agonize": 347, | |
| "wool": 348, | |
| "muggy": 349, | |
| "ham": 350, | |
| "aqua": 351, | |
| "palmy": 352, | |
| "found": 353, | |
| "produce": 354, | |
| "leach": 355, | |
| "doing": 356, | |
| "wits": 357, | |
| "forum": 358, | |
| "grain": 359, | |
| "legitimise": 360, | |
| "spoil": 361, | |
| "walk": 362, | |
| "indeed": 363, | |
| "clothes": 364, | |
| "soak": 365, | |
| "sixty": 366, | |
| "using": 367, | |
| "babble": 368, | |
| "hail": 369, | |
| "ordinary": 370, | |
| "shot": 371, | |
| "ten": 372, | |
| "barn": 373, | |
| "gulch": 374, | |
| "seat": 375, | |
| "during": 376, | |
| "acme": 377, | |
| "vial": 378, | |
| "herald": 379, | |
| "tomorrow": 380, | |
| "fudge": 381, | |
| "mice": 382, | |
| "lowly": 383, | |
| "goon": 384, | |
| "ace": 385, | |
| "rake": 386 | |
| }, | |
| "unk_token": "[UNK]" | |
| } | |
| } |