Tokenizer-Visualizer / index.html
quickgrid's picture
v1.4
33ff8e6 verified
Raw History Blame Contribute Delete
60.4 kB
<!DOCTYPE html> <html lang="en" data-theme="dark"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> <title>TokenLens β€” LLM Tokenizer Playground</title> <meta name="description" content="Visualize how large language models tokenize text. Powered by Transformers.js, runs entirely in your browser." /> <link rel="preconnect" href="https://fonts.googleapis.com" /> <link href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:opsz,wght@12..96,300;12..96,400;12..96,500;12..96,600;12..96,700;12..96,800&family=JetBrains+Mono:wght@300;400;500;700&family=DM+Sans:wght@300;400;500&display=swap" rel="stylesheet" /> <style> /* ─── Design Tokens ─────────────────────────────────── */ :root { --bg: #060b14; --bg2: #0b1220; --bg3: #101828; --bg4: #162035; --border: #1a2d4a; --border2: #243d60; --glow: #1f3d6e; --text: #dce8f8; --text2: #7899c0; --text3: #3d5a80; --accent: #4d9ef5; --accent2: #8b6af5; --green: #34d89a; --amber: #f5a623; --red: #f55577; } [data-theme="light"] { --bg: #eef1f8; --bg2: #e2e7f2; --bg3: #d3dae8; --bg4: #c2cce0; --border: #b8c4d8; --border2: #a0aec8; --glow: #c0d0e8; --text: #1a2236; --text2: #5a6888; --text3: #8898b4; --accent: #2878e0; --accent2: #6838d8; --green: #18a060; --amber: #c88010; --red: #d83858; } /* ─── Reset ─────────────────────────────────────────── */ *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } html { height: 100%; overflow: hidden; } body { background: var(--bg); color: var(--text); font-family: 'DM Sans', sans-serif; height: 100%; overflow: hidden; } /* ─── Background FX ─────────────────────────────────── */ .bg-gradient { position: fixed; inset: 0; pointer-events: none; z-index: 0; background: radial-gradient(ellipse 80% 50% at 20% 10%, rgba(77,158,245,.06) 0%, transparent 70%), radial-gradient(ellipse 60% 40% at 80% 90%, rgba(139,106,245,.05) 0%, transparent 60%), radial-gradient(ellipse 40% 30% at 60% 50%, rgba(52,216,154,.03) 0%, transparent 60%); } [data-theme="light"] .bg-gradient { background: radial-gradient(ellipse 80% 50% at 20% 10%, rgba(40,120,224,.05) 0%, transparent 70%), radial-gradient(ellipse 60% 40% at 80% 90%, rgba(104,56,216,.04) 0%, transparent 60%), radial-gradient(ellipse 40% 30% at 60% 50%, rgba(24,160,96,.02) 0%, transparent 60%); } .dot-grid { position: fixed; inset: 0; pointer-events: none; z-index: 0; background-image: radial-gradient(circle, rgba(77,158,245,.12) 1px, transparent 1px); background-size: 36px 36px; mask-image: radial-gradient(ellipse 100% 100% at 50% 50%, black 30%, transparent 80%); } [data-theme="light"] .dot-grid { background-image: radial-gradient(circle, rgba(40,120,224,.06) 1px, transparent 1px); } /* ─── Layout ─────────────────────────────────────────── */ #app { position: relative; z-index: 1; display: flex; flex-direction: column; height: 100vh; overflow: hidden; } /* ─── Header ─────────────────────────────────────────── */ header { display: flex; align-items: center; padding: 0 20px; height: 56px; border-bottom: 1px solid var(--border); background: rgba(6,11,20,.85); backdrop-filter: blur(20px); flex-shrink: 0; z-index: 100; gap: 12px; } [data-theme="light"] header { background: rgba(238,241,248,.92); } .logo { display: flex; align-items: center; gap: 8px; text-decoration: none; color: var(--text); flex-shrink: 0; } .logo-hex { width: 30px; height: 30px; background: linear-gradient(135deg, var(--accent), var(--accent2)); clip-path: polygon(50% 0%, 93% 25%, 93% 75%, 50% 100%, 7% 75%, 7% 25%); display: flex; align-items: center; justify-content: center; font-size: 12px; font-family: 'JetBrains Mono', monospace; font-weight: 700; color: white; } .logo-name { font-family: 'Bricolage Grotesque', sans-serif; font-size: 17px; font-weight: 700; letter-spacing: -0.5px; background: linear-gradient(135deg, #dce8f8 40%, var(--accent)); -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text; } [data-theme="light"] .logo-name { background: linear-gradient(135deg, #1a2236 40%, var(--accent)); -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text; } .logo-tag { font-size: 9px; font-family: 'JetBrains Mono', monospace; color: var(--text3); background: var(--bg3); border: 1px solid var(--border); padding: 1px 5px; border-radius: 4px; letter-spacing: .5px; } .header-divider { width: 1px; height: 28px; background: var(--border); flex-shrink: 0; } /* ─── Search Bar Groups ─────────────────────────────── */ .header-controls { display: flex; align-items: center; gap: 10px; flex: 1; min-width: 0; } .searchbar-group { position: relative; display: flex; align-items: center; gap: 3px; flex: 1; min-width: 0; } .searchbar-label { font-family: 'JetBrains Mono', monospace; font-size: 11px; font-weight: 700; width: 22px; height: 22px; border-radius: 5px; display: flex; align-items: center; justify-content: center; flex-shrink: 0; border: 1px solid var(--border); } .searchbar-label.label-a { background: rgba(77,158,245,.15); color: var(--accent); border-color: rgba(77,158,245,.3); } .searchbar-label.label-b { background: rgba(139,106,245,.15); color: var(--accent2); border-color: rgba(139,106,245,.3); } .searchbar-input { flex: 1; min-width: 0; background: var(--bg2); border: 1px solid var(--border); border-radius: 6px; color: var(--text); font-family: 'JetBrains Mono', monospace; font-size: 11px; padding: 5px 8px; outline: none; transition: border-color .2s; } .searchbar-input:focus { border-color: var(--accent); } .searchbar-input::placeholder { color: var(--text3); } .searchbar-dropdown-btn { width: 24px; height: 24px; border-radius: 5px; border: 1px solid var(--border); background: var(--bg2); color: var(--text2); cursor: pointer; font-size: 10px; display: flex; align-items: center; justify-content: center; flex-shrink: 0; transition: all .15s; } .searchbar-dropdown-btn:hover { border-color: var(--border2); color: var(--text); } .searchbar-load-btn { padding: 4px 10px; border-radius: 5px; border: 1px solid var(--border2); background: linear-gradient(135deg, rgba(77,158,245,.12), rgba(139,106,245,.12)); color: var(--accent); font-family: 'DM Sans', sans-serif; font-size: 11px; font-weight: 500; cursor: pointer; transition: all .15s; white-space: nowrap; flex-shrink: 0; } .searchbar-load-btn:hover { background: linear-gradient(135deg, rgba(77,158,245,.22), rgba(139,106,245,.22)); border-color: var(--accent); } /* ─── Dropdown Menu ─────────────────────────────────── */ .dropdown-menu { position: absolute; top: calc(100% + 6px); left: 22px; right: 0; min-width: 280px; max-height: 320px; overflow-y: auto; background: var(--bg3); border: 1px solid var(--border2); border-radius: 8px; z-index: 200; display: none; padding: 4px; box-shadow: 0 8px 32px rgba(0,0,0,.4); } [data-theme="light"] .dropdown-menu { box-shadow: 0 8px 32px rgba(0,0,0,.12); } .dropdown-menu.open { display: block; } .dropdown-item { padding: 7px 10px; cursor: pointer; border-radius: 5px; display: flex; align-items: center; gap: 8px; font-size: 12px; transition: background .12s; } .dropdown-item:hover { background: var(--bg4); } .dropdown-item-dot { width: 6px; height: 6px; border-radius: 50%; flex-shrink: 0; } .dropdown-item-name { font-weight: 600; color: var(--text); font-family: 'Bricolage Grotesque', sans-serif; } .dropdown-item-detail { font-size: 10px; color: var(--text3); font-family: 'JetBrains Mono', monospace; margin-left: auto; white-space: nowrap; } /* ─── Icon Toggle Buttons ────────────────────────────── */ .icon-toggle-btn { width: 24px; height: 24px; border-radius: 7px; border: 1px solid var(--border); background: var(--bg2); color: var(--text2); cursor: pointer; flex-shrink: 0; display: flex; align-items: center; justify-content: center; transition: all .15s; } .icon-toggle-btn:hover { border-color: var(--border2); color: var(--text); } .icon-toggle-btn.active { border-color: var(--accent); color: var(--accent); background: rgba(77,158,245,.1); } .icon-toggle-btn svg { width: 16px; height: 16px; } /* ─── Mobile Tab Bar ─────────────────────────────────── */ .mobile-tab-bar { display: none; border-bottom: 1px solid var(--border); flex-shrink: 0; background: var(--bg2); padding: 0 4px; } .mobile-tab { flex: 1; padding: 10px 4px; border: none; background: transparent; color: var(--text2); font-family: 'DM Sans', sans-serif; font-size: 12px; font-weight: 500; cursor: pointer; border-bottom: 2px solid transparent; transition: all .15s; text-align: center; } .mobile-tab.active { color: var(--accent); border-bottom-color: var(--accent); } .mobile-tab.tab-disabled { opacity: 0.3; pointer-events: none; } /* ─── Main Split ─────────────────────────────────────── */ main { flex: 1; min-height: 0; overflow: hidden; display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 0; } main.single-panel { grid-template-columns: 1fr 2fr; } /* ─── Left Panel (Input) ─────────────────────────────── */ .input-panel { border-right: 1px solid var(--border); display: flex; flex-direction: column; overflow: hidden; } .panel-header { padding: 7px 16px 7px; border-bottom: 1px solid var(--border); display: flex; align-items: center; justify-content: space-between; flex-shrink: 0; } .panel-title { font-family: 'Bricolage Grotesque', sans-serif; font-size: 13px; font-weight: 600; color: var(--text2); letter-spacing: .3px; display: flex; align-items: center; gap: 6px; } .panel-title-icon { width: 18px; height: 18px; background: var(--bg4); border: 1px solid var(--border); border-radius: 4px; display: flex; align-items: center; justify-content: center; font-size: 10px; } .sample-btns { display: flex; gap: 4px; flex-wrap: wrap; } .sample-btn { font-size: 10px; padding: 3px 8px; border-radius: 5px; border: 1px solid var(--border); background: var(--bg2); color: var(--text2); cursor: pointer; font-family: 'DM Sans', sans-serif; transition: all .15s; } .sample-btn:hover { border-color: var(--border2); color: var(--text); } #input-area { flex: 1; width: 100%; background: transparent; border: none; outline: none; resize: none; color: var(--text); font-family: 'DM Sans', sans-serif; font-size: 14px; line-height: 1.7; padding: 14px 16px; min-height: 0; overflow-y: auto; } #input-area::placeholder { color: var(--text3); } .char-counter { padding: 6px 16px; border-top: 1px solid var(--border); flex-shrink: 0; font-size: 10px; font-family: 'JetBrains Mono', monospace; color: var(--text3); text-align: right; } /* ─── Output Panel ───────────────────────────────────── */ .output-panel { display: flex; flex-direction: column; overflow: hidden; min-height: 0; } .output-panel + .output-panel { border-left: 1px solid var(--border); } .output-panel-header { padding: 10px 14px; border-bottom: 1px solid var(--border); display: flex; align-items: center; justify-content: space-between; flex-shrink: 0; gap: 8px; } .model-indicator { display: flex; align-items: center; gap: 5px; font-size: 11px; font-family: 'JetBrains Mono', monospace; color: var(--text2); min-width: 0; overflow: hidden; } .model-indicator-dot { width: 7px; height: 7px; border-radius: 50%; flex-shrink: 0; } .model-indicator-name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } /* Stats row */ .stats-row { display: grid; grid-template-columns: 1fr 1fr; border-bottom: 1px solid var(--border); flex-shrink: 0; } .stat-card { padding: 10px 14px; border-right: 1px solid var(--border); border-bottom: 1px solid var(--border); position: relative; overflow: hidden; } .stat-card:nth-child(2n) { border-right: none; } .stat-card:nth-child(3), .stat-card:nth-child(4) { border-bottom: none; } .stat-card::after { content: ''; position: absolute; bottom: 0; left: 0; right: 0; height: 2px; background: linear-gradient(90deg, transparent, var(--accent), transparent); opacity: 0; transition: opacity .3s; } .stat-card.highlight::after { opacity: 1; } .stat-label { font-size: 9px; font-family: 'JetBrains Mono', monospace; color: var(--text3); text-transform: uppercase; letter-spacing: 1px; margin-bottom: 4px; } .stat-value { font-family: 'Bricolage Grotesque', sans-serif; font-size: 20px; font-weight: 700; color: var(--text); line-height: 1; transition: all .3s; } .stat-card:nth-child(1) .stat-value { color: var(--accent); } .stat-card:nth-child(2) .stat-value { color: var(--green); } .stat-card:nth-child(3) .stat-value { color: var(--amber); } .stat-card:nth-child(4) .stat-value { color: var(--accent2); } .stat-sub { font-size: 9px; color: var(--text3); font-family: 'JetBrains Mono', monospace; margin-top: 2px; } /* View toggle */ .view-toggle { display: flex; padding: 8px 14px; border-bottom: 1px solid var(--border); gap: 4px; align-items: center; justify-content: space-between; flex-shrink: 0; } .toggle-group { display: flex; gap: 2px; background: var(--bg2); border: 1px solid var(--border); border-radius: 6px; padding: 2px; } .toggle-btn { padding: 3px 10px; border-radius: 4px; border: none; background: transparent; color: var(--text2); font-family: 'DM Sans', sans-serif; font-size: 11px; font-weight: 500; cursor: pointer; transition: all .15s; } .toggle-btn.active { background: var(--bg4); color: var(--text); box-shadow: 0 1px 4px rgba(0,0,0,.3); } /* Token Display */ .token-display { flex: 1; overflow-y: auto; padding: 14px; min-height: 0; scrollbar-width: thin; scrollbar-color: var(--border) transparent; } .placeholder-msg { display: flex; flex-direction: column; align-items: center; justify-content: center; height: 100%; min-height: 120px; gap: 12px; color: var(--text3); } .placeholder-icon { font-size: 32px; filter: grayscale(1) opacity(.3); } .placeholder-msg p { font-family: 'JetBrains Mono', monospace; font-size: 11px; text-align: center; line-height: 1.6; } /* ─── Token Visualization Views ───────────────────────── */ .token-text-view { font-family: 'JetBrains Mono', monospace; font-size: 13px; line-height: 2.2; word-break: break-all; } .tok { display: inline; border-radius: 3px; padding: 1px 0; cursor: default; transition: filter .15s; position: relative; } .tok:hover { filter: brightness(1.3); } .tok-tooltip { display: none; position: absolute; bottom: 110%; left: 50%; transform: translateX(-50%); background: var(--bg4); border: 1px solid var(--border2); border-radius: 5px; padding: 4px 7px; font-size: 10px; white-space: nowrap; z-index: 50; pointer-events: none; box-shadow: 0 4px 20px rgba(0,0,0,.5); } [data-theme="light"] .tok-tooltip { box-shadow: 0 4px 16px rgba(0,0,0,.12); } .tok:hover .tok-tooltip { display: block; } .tok-tooltip-id { color: var(--accent); font-weight: 700; } .tok-tooltip-text { color: var(--text2); } .tok-space::before { content: 'Β·'; opacity: .3; } .tok-newline::before { content: '↡'; opacity: .5; } /* ID VIEW */ .token-id-view { display: flex; flex-wrap: wrap; gap: 5px; } .tok-id-card { display: flex; flex-direction: column; align-items: center; border-radius: 6px; overflow: hidden; border: 1px solid; cursor: default; transition: transform .15s, box-shadow .15s; min-width: 46px; } .tok-id-card:hover { transform: translateY(-2px); box-shadow: 0 4px 16px rgba(0,0,0,.4); } .tok-id-top { padding: 2px 5px; font-family: 'JetBrains Mono', monospace; font-size: 10px; font-weight: 500; width: 100%; text-align: center; border-bottom: 1px solid rgba(255,255,255,.08); } [data-theme="light"] .tok-id-top { border-bottom-color: rgba(0,0,0,.06); } .tok-id-bottom { padding: 1px 5px 2px; font-family: 'JetBrains Mono', monospace; font-size: 8px; color: rgba(255,255,255,.4); width: 100%; text-align: center; } [data-theme="light"] .tok-id-bottom { color: rgba(0,0,0,.35); } /* LIST VIEW */ .token-split-view { display: flex; flex-direction: column; gap: 2px; } .tok-split-row { display: flex; align-items: stretch; border-radius: 5px; overflow: hidden; border: 1px solid; font-family: 'JetBrains Mono', monospace; font-size: 11px; } .tok-split-idx { width: 34px; text-align: center; padding: 4px 3px; font-size: 9px; color: rgba(255,255,255,.3); border-right: 1px solid rgba(255,255,255,.06); display: flex; align-items: center; justify-content: center; } [data-theme="light"] .tok-split-idx { color: rgba(0,0,0,.25); border-right-color: rgba(0,0,0,.06); } .tok-split-text { flex: 1; padding: 4px 6px; font-size: 12px; } .tok-split-id { padding: 4px 6px; font-size: 10px; color: rgba(255,255,255,.45); border-left: 1px solid rgba(255,255,255,.06); display: flex; align-items: center; } [data-theme="light"] .tok-split-id { color: rgba(0,0,0,.35); border-left-color: rgba(0,0,0,.06); } /* ─── Loading Overlay ────────────────────────────────── */ #loading-overlay { position: fixed; inset: 0; background: rgba(6,11,20,.92); backdrop-filter: blur(8px); z-index: 1000; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 20px; transition: opacity .4s; } [data-theme="light"] #loading-overlay { background: rgba(238,241,248,.92); } #loading-overlay.hidden { opacity: 0; pointer-events: none; } .loading-spinner { width: 48px; height: 48px; position: relative; } .loading-spinner::before, .loading-spinner::after { content: ''; position: absolute; border-radius: 50%; border: 2px solid transparent; } .loading-spinner::before { inset: 0; border-top-color: var(--accent); animation: spin 1s linear infinite; } .loading-spinner::after { inset: 7px; border-top-color: var(--accent2); animation: spin .7s linear infinite reverse; } @keyframes spin { to { transform: rotate(360deg); } } .loading-text { font-family: 'Bricolage Grotesque', sans-serif; font-size: 18px; font-weight: 600; color: var(--text); } .loading-sub { font-family: 'JetBrains Mono', monospace; font-size: 11px; color: var(--text2); max-width: 340px; text-align: center; } .loading-bar-wrap { width: 260px; height: 3px; background: var(--bg3); border-radius: 2px; overflow: hidden; } .loading-bar { height: 100%; width: 0%; background: linear-gradient(90deg, var(--accent), var(--accent2)); border-radius: 2px; transition: width .3s; } .loading-file { font-size: 10px; font-family: 'JetBrains Mono', monospace; color: var(--text3); } /* ─── Error Toast ────────────────────────────────────── */ #toast { position: fixed; bottom: 24px; left: 50%; transform: translateX(-50%) translateY(80px); background: rgba(245,85,119,.15); border: 1px solid rgba(245,85,119,.4); color: var(--red); padding: 8px 18px; border-radius: 8px; font-size: 12px; font-family: 'JetBrains Mono', monospace; z-index: 500; transition: transform .3s; max-width: 460px; text-align: center; } #toast.show { transform: translateX(-50%) translateY(0); } /* ─── Footer ─────────────────────────────────────────── */ footer { padding: 8px 24px; border-top: 1px solid var(--border); display: flex; align-items: center; justify-content: space-between; font-size: 10px; color: var(--text3); font-family: 'JetBrains Mono', monospace; background: rgba(6,11,20,.8); flex-shrink: 0; } [data-theme="light"] footer { background: rgba(238,241,248,.8); } footer a { color: var(--text2); text-decoration: none; transition: color .15s; } footer a:hover { color: var(--accent); } /* ─── Scrollbar ──────────────────────────────────────── */ ::-webkit-scrollbar { width: 5px; } ::-webkit-scrollbar-track { background: transparent; } ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; } ::-webkit-scrollbar-thumb:hover { background: var(--border2); } /* ─── Animations ─────────────────────────────────────── */ @keyframes fadeIn { from { opacity: 0; transform: translateY(4px); } to { opacity: 1; transform: translateY(0); } } .fade-in { animation: fadeIn .2s ease forwards; } /* ─── Responsive: Tablet (600–900px) ──────────────────── */ @media (min-width: 601px) and (max-width: 900px) { header { flex-wrap: wrap; height: auto; padding: 8px 12px; gap: 8px; } .header-divider { display: none; } .header-controls { flex-basis: 100%; display: flex; flex-wrap: wrap; gap: 8px; } .searchbar-group { flex: 1 1 180px; } main { grid-template-columns: 1fr; grid-template-rows: auto 1fr 1fr; } main.single-panel { grid-template-columns: 1fr; grid-template-rows: auto 1fr; } .input-panel { border-right: none; border-bottom: 1px solid var(--border); max-height: 25vh; } .output-panel { border-left: none !important; border-bottom: 1px solid var(--border); } .output-panel:last-child { border-bottom: none; } .stat-card { padding: 7px 10px; } .stat-value { font-size: 17px; } .stat-sub { font-size: 8px; } .stat-label { font-size: 8px; } .output-panel-header { padding: 8px 10px; } .view-toggle { padding: 6px 10px; } } /* ─── Responsive: Mobile (≀600px) ────────────────────── */ @media (max-width: 600px) { header { flex-wrap: wrap; height: auto; padding: 6px 10px; gap: 6px; } .header-divider { display: none; } .logo-tag { display: none; } .header-controls { flex-basis: 100%; display: flex; flex-wrap: wrap; gap: 6px; } .searchbar-group { flex: 1 1 100%; } .searchbar-label { display: none; } .dropdown-menu { left: 0 !important; right: 0 !important; min-width: unset; } .mobile-tab-bar { display: flex; } main { grid-template-columns: 1fr; grid-template-rows: 1fr; } .input-panel, .output-panel { display: none; } .mobile-active { display: flex !important; } .input-panel { border-right: none; border-bottom: none; max-height: none; } .output-panel { border-left: none !important; border-bottom: none; } .panel-header { flex-wrap: wrap; gap: 6px; padding: 10px 12px 8px; } .sample-btns { width: 100%; } .sample-btn { font-size: 9px; padding: 3px 6px; } .output-panel-header { padding: 6px 10px; } .model-indicator { font-size: 10px; } .stat-card { padding: 5px 8px; } .stat-value { font-size: 16px; } .stat-label { font-size: 7px; letter-spacing: .5px; margin-bottom: 2px; } .stat-sub { font-size: 7px; } .view-toggle { padding: 5px 10px; } .toggle-btn { padding: 3px 8px; font-size: 10px; } .token-display { padding: 10px; } .token-text-view { font-size: 12px; line-height: 2; } .tok-split-idx { width: 28px; font-size: 8px; } .tok-split-text { font-size: 11px; } .tok-split-id { font-size: 9px; } footer { padding: 6px 12px; font-size: 9px; } footer span:last-child { display: none; } } </style> </head> <body> <div class="bg-gradient"></div> <div class="dot-grid"></div> <div id="app"> <!-- Header --> <header> <div class="logo"> <div class="logo-hex">T</div> <span class="logo-name">TokenLens</span> <span class="logo-tag">v1.4</span> </div> <div class="header-divider"></div> <div class="header-controls"> <!-- Search bar A --> <div class="searchbar-group" id="search-group-0"> <span class="searchbar-label label-a">A</span> <input class="searchbar-input" id="search-input-0" type="text" placeholder="Model A: HF repo id…" /> <button class="searchbar-dropdown-btn" id="dropdown-btn-0" title="Predefined models"> <svg width="10" height="10" viewBox="0 0 10 10" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M2 3.5L5 6.5L8 3.5"/></svg> </button> <button class="searchbar-load-btn" id="load-btn-0">Load</button> <div class="dropdown-menu" id="dropdown-menu-0"></div> </div> <!-- Search bar B --> <div class="searchbar-group" id="search-group-1"> <span class="searchbar-label label-b">B</span> <input class="searchbar-input" id="search-input-1" type="text" placeholder="Model B: HF repo id…" /> <button class="searchbar-dropdown-btn" id="dropdown-btn-1" title="Predefined models"> <svg width="10" height="10" viewBox="0 0 10 10" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M2 3.5L5 6.5L8 3.5"/></svg> </button> <button class="searchbar-load-btn" id="load-btn-1">Load</button> <div class="dropdown-menu" id="dropdown-menu-1"></div> </div> <!-- Toggle: show/hide panel B --> <button class="icon-toggle-btn active" id="panel-toggle" title="Toggle comparison panel"> <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><rect x="3" y="3" width="7" height="18" rx="1"/><rect x="14" y="3" width="7" height="18" rx="1"/></svg> </button> <!-- Toggle: light/dark theme --> <button class="icon-toggle-btn" id="theme-toggle" title="Toggle light/dark theme"> <svg id="theme-icon-sun" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" style="display:none"><circle cx="12" cy="12" r="5"/><line x1="12" y1="1" x2="12" y2="3"/><line x1="12" y1="21" x2="12" y2="23"/><line x1="4.22" y1="4.22" x2="5.64" y2="5.64"/><line x1="18.36" y1="18.36" x2="19.78" y2="19.78"/><line x1="1" y1="12" x2="3" y2="12"/><line x1="21" y1="12" x2="23" y2="12"/><line x1="4.22" y1="19.78" x2="5.64" y2="18.36"/><line x1="18.36" y1="5.64" x2="19.78" y2="4.22"/></svg> <svg id="theme-icon-moon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z"/></svg> </button> </div> </header> <!-- Mobile Tab Bar --> <div class="mobile-tab-bar" id="mobile-tab-bar"> <button class="mobile-tab active" data-tab="input">Input</button> <button class="mobile-tab" data-tab="panel-0">Model A</button> <button class="mobile-tab" data-tab="panel-1">Model B</button> </div> <!-- Main --> <main id="main-grid"> <!-- Left: Input --> <div class="input-panel mobile-active" id="input-panel"> <div class="panel-header"> <div class="panel-title"> <div class="panel-title-icon">✎</div> Input Text </div> <div class="sample-btns"> <button class="sample-btn" data-sample="poetry">Poetry</button> <button class="sample-btn" data-sample="code">Code</button> <button class="sample-btn" data-sample="multilingual">Multi</button> <button class="sample-btn" data-sample="numbers">Numbers</button> <button class="sample-btn" data-sample="clear">Clear</button> </div> </div> <textarea id="input-area" placeholder="Type or paste text here to see how tokenizers split it into tokens… Try special characters, code, emojis, or multi-lingual text to compare models."></textarea> <div class="char-counter"><span id="char-count">0</span> characters</div> </div> <!-- Visualizer A --> <div class="output-panel" id="panel-0"> <div class="output-panel-header"> <div class="model-indicator" id="model-indicator-0"> <div class="model-indicator-dot" id="model-dot-0" style="background:#3d5a80"></div> <span class="model-indicator-name" id="model-label-0">A: no model</span> </div> </div> <div class="stats-row"> <div class="stat-card" id="sc-tokens-0"> <div class="stat-label">Tokens</div> <div class="stat-value" id="stat-tokens-0">β€”</div> <div class="stat-sub" id="stat-model-0">no model loaded</div> </div> <div class="stat-card" id="sc-chars-0"> <div class="stat-label">Characters</div> <div class="stat-value" id="stat-chars-0">β€”</div> <div class="stat-sub">total input</div> </div> <div class="stat-card" id="sc-words-0"> <div class="stat-label">Words</div> <div class="stat-value" id="stat-words-0">β€”</div> <div class="stat-sub">approx</div> </div> <div class="stat-card" id="sc-ratio-0"> <div class="stat-label">Chars/Token</div> <div class="stat-value" id="stat-ratio-0">β€”</div> <div class="stat-sub">efficiency</div> </div> </div> <div class="view-toggle"> <div class="toggle-group" id="toggle-group-0"> <button class="toggle-btn active" data-view="text" data-panel="0">Text View</button> <button class="toggle-btn" data-view="ids" data-panel="0">ID Grid</button> <button class="toggle-btn" data-view="list" data-panel="0">Token List</button> </div> </div> <div class="token-display" id="token-display-0"> <div class="placeholder-msg" id="placeholder-0"> <div class="placeholder-icon">⬑</div> <p>Load a tokenizer using search bar A above<br>then type text to see tokenization</p> </div> </div> </div> <!-- Visualizer B --> <div class="output-panel" id="panel-1"> <div class="output-panel-header"> <div class="model-indicator" id="model-indicator-1"> <div class="model-indicator-dot" id="model-dot-1" style="background:#3d5a80"></div> <span class="model-indicator-name" id="model-label-1">B: no model</span> </div> </div> <div class="stats-row"> <div class="stat-card" id="sc-tokens-1"> <div class="stat-label">Tokens</div> <div class="stat-value" id="stat-tokens-1">β€”</div> <div class="stat-sub" id="stat-model-1">no model loaded</div> </div> <div class="stat-card" id="sc-chars-1"> <div class="stat-label">Characters</div> <div class="stat-value" id="stat-chars-1">β€”</div> <div class="stat-sub">total input</div> </div> <div class="stat-card" id="sc-words-1"> <div class="stat-label">Words</div> <div class="stat-value" id="stat-words-1">β€”</div> <div class="stat-sub">approx</div> </div> <div class="stat-card" id="sc-ratio-1"> <div class="stat-label">Chars/Token</div> <div class="stat-value" id="stat-ratio-1">β€”</div> <div class="stat-sub">efficiency</div> </div> </div> <div class="view-toggle"> <div class="toggle-group" id="toggle-group-1"> <button class="toggle-btn active" data-view="text" data-panel="1">Text View</button> <button class="toggle-btn" data-view="ids" data-panel="1">ID Grid</button> <button class="toggle-btn" data-view="list" data-panel="1">Token List</button> </div> </div> <div class="token-display" id="token-display-1"> <div class="placeholder-msg" id="placeholder-1"> <div class="placeholder-icon">⬑</div> <p>Load a tokenizer using search bar B above<br>then type text to see tokenization</p> </div> </div> </div> </main> <footer> <span>TokenLens β€” Powered by <a href="https://github.com/xenova/transformers.js" target="_blank">Transformers.js</a> Β· Runs entirely in your browser</span> <span><a href="https://quickgrid.github.io/">Made by Β· Asif Ahmed</a></span> </footer> </div> <!-- Loading Overlay --> <div id="loading-overlay"> <div class="loading-spinner"></div> <div class="loading-text" id="loading-title">Loading Tokenizer</div> <div class="loading-sub" id="loading-sub">Downloading tokenizer files from Hugging Face Hub…<br>Cached in your browser after first download.</div> <div class="loading-bar-wrap"><div class="loading-bar" id="loading-bar"></div></div> <div class="loading-file" id="loading-file"></div> </div> <!-- Toast --> <div id="toast"></div> <script type="module"> import { AutoTokenizer, env } from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0'; env.allowLocalModels = false; env.useBrowserCache = true; // ── Model Registry ───────────────────────────────────────── const MODELS = [ { id:'Qwen/Qwen3.6-27B', name:'Qwen 3.6 27B', org:'Alibaba', color:'#0466de', vocab:'152k', type:'BPE', desc:'Qwen tokenizer optimized for multilingual coding and reasoning tasks' }, { id:'deepseek-ai/DeepSeek-V4-Pro', name:'DeepSeek V4 Pro', org:'DeepSeek', color:'#4285f4', vocab:'129k', type:'BPE', desc:'DeepSeek multilingual tokenizer designed for code-heavy workloads' }, { id:'MiniMaxAI/MiniMax-M2.7', name:'MiniMax M2.7', org:'MiniMax', color:'#1a73e8', vocab:'128k', type:'SentencePiece', desc:'Efficient multilingual tokenizer for long-context multimodal models' }, { id:'mistralai/Mistral-Medium-3.5-128B', name:'Mistral Medium 3.5', org:'Mistral AI', color:'#ff7722', vocab:'131k', type:'Tekken BPE', desc:'Mistral Tekken tokenizer with efficient multilingual compression' }, { id:'google/gemma-4-31B-it', name:'Gemma 4 31B', org:'Google', color:'#34a853', vocab:'256k', type:'SentencePiece', desc:'Gemma multilingual SentencePiece tokenizer with large vocabulary' }, { id:'zai-org/GLM-5.1', name:'GLM-5.1', org:'Z.ai', color:'#10a37f', vocab:'151k', type:'SentencePiece', desc:'GLM multilingual SentencePiece tokenizer with large bilingual vocabulary' }, { id:'XiaomiMiMo/MiMo-V2.5-Pro', name:'MiMo V2.5 Pro', org:'Xiaomi', color:'#ff7722', vocab:'128k', type:'SentencePiece', desc:'MoE tokenizer tuned for multilingual reasoning and code generation' }, ]; // ── Token Color Palettes ─────────────────────────────────── const PALETTE_DARK = [ { text:'#ff8080', bg:'rgba(255,128,128,.18)', border:'rgba(255,128,128,.35)' }, { text:'#ffb84d', bg:'rgba(255,184, 77,.18)', border:'rgba(255,184, 77,.35)' }, { text:'#ffe066', bg:'rgba(255,224,102,.18)', border:'rgba(255,224,102,.35)' }, { text:'#7aed91', bg:'rgba(122,237,145,.18)', border:'rgba(122,237,145,.35)' }, { text:'#4ddfc0', bg:'rgba( 77,223,192,.18)', border:'rgba( 77,223,192,.35)' }, { text:'#56c8f5', bg:'rgba( 86,200,245,.18)', border:'rgba( 86,200,245,.35)' }, { text:'#748ef8', bg:'rgba(116,142,248,.18)', border:'rgba(116,142,248,.35)' }, { text:'#c484f8', bg:'rgba(196,132,248,.18)', border:'rgba(196,132,248,.35)' }, { text:'#f57cd4', bg:'rgba(245,124,212,.18)', border:'rgba(245,124,212,.35)' }, { text:'#fa8072', bg:'rgba(250,128,114,.18)', border:'rgba(250,128,114,.35)' }, { text:'#8be08b', bg:'rgba(139,224,139,.18)', border:'rgba(139,224,139,.35)' }, { text:'#f0c040', bg:'rgba(240,192, 64,.18)', border:'rgba(240,192, 64,.35)' }, { text:'#60d4e0', bg:'rgba( 96,212,224,.18)', border:'rgba( 96,212,224,.35)' }, { text:'#e89060', bg:'rgba(232,144, 96,.18)', border:'rgba(232,144, 96,.35)' }, ]; const PALETTE_LIGHT = [ { text:'#cc3333', bg:'rgba(204, 51, 51,.12)', border:'rgba(204, 51, 51,.25)' }, { text:'#b87218', bg:'rgba(184,114, 24,.12)', border:'rgba(184,114, 24,.25)' }, { text:'#a08618', bg:'rgba(160,134, 24,.12)', border:'rgba(160,134, 24,.25)' }, { text:'#228838', bg:'rgba( 34,136, 56,.12)', border:'rgba( 34,136, 56,.25)' }, { text:'#1a8870', bg:'rgba( 26,136,112,.12)', border:'rgba( 26,136,112,.25)' }, { text:'#1890b8', bg:'rgba( 24,144,184,.12)', border:'rgba( 24,144,184,.25)' }, { text:'#3850b8', bg:'rgba( 56, 80,184,.12)', border:'rgba( 56, 80,184,.25)' }, { text:'#7830a8', bg:'rgba(120, 48,168,.12)', border:'rgba(120, 48,168,.25)' }, { text:'#b03088', bg:'rgba(176, 48,136,.12)', border:'rgba(176, 48,136,.25)' }, { text:'#b83828', bg:'rgba(184, 56, 40,.12)', border:'rgba(184, 56, 40,.25)' }, { text:'#2a882a', bg:'rgba( 42,136, 42,.12)', border:'rgba( 42,136, 42,.25)' }, { text:'#9a7018', bg:'rgba(154,112, 24,.12)', border:'rgba(154,112, 24,.25)' }, { text:'#1a8898', bg:'rgba( 26,136,152,.12)', border:'rgba( 26,136,152,.25)' }, { text:'#a05020', bg:'rgba(160, 80, 32,.12)', border:'rgba(160, 80, 32,.25)' }, ]; function getPalette() { return document.documentElement.dataset.theme === 'light' ? PALETTE_LIGHT : PALETTE_DARK; } // ── Sample texts ─────────────────────────────────────────── const SAMPLES = { poetry: `Two roads diverged in a yellow wood,\nAnd sorry I could not travel both\nAnd be one traveler, long I stood\nAnd looked down one as far as I could\nTo where it bent in the undergrowth;\nβ€” Robert Frost, "The Road Not Taken"`, code: `async function fetchData(url, retries = 3) {\n for (let i = 0; i < retries; i++) {\n try {\n const res = await fetch(url);\n if (!res.ok) throw new Error(\`HTTP \${res.status}\`);\n return await res.json();\n } catch (e) {\n if (i === retries - 1) throw e;\n await new Promise(r => setTimeout(r, 1000 * 2 ** i));\n }\n }\n}`, multilingual: `English: The quick brown fox jumps over the lazy dog.\nζ—₯本θͺž: εΎθΌ©γ―ηŒ«γ§γ‚γ‚‹γ€‚εε‰γ―γΎγ γͺい。\nδΈ­ζ–‡: ζ˜₯ηœ δΈθ§‰ζ™“οΌŒε€„ε€„ι—»ε•ΌιΈŸγ€‚\nΨ§Ω„ΨΉΨ±Ψ¨ΩŠΨ©: Ψ§Ω„Ω„ΨΊΨ© Ψ§Ω„ΨΉΨ±Ψ¨ΩŠΨ© Ψ¬Ω…ΩŠΩ„Ψ© ΩˆΩ…ΨΉΨ¨Ψ±Ψ©.\nΕλληνικά: Ξ— Ξ³Ξ½ΟŽΟƒΞ· Ρίναι δύναμη.\nEmoji: 🌍 🦊 ⚑ 🎯 🧬 πŸ€– πŸ¦‹`, numbers: `Ο€ β‰ˆ 3.14159265358979323846\ne β‰ˆ 2.71828182845904523536\nΟ† β‰ˆ 1.61803398874989484820\n1,000,000 Γ— $42.99 = $42,990,000.00\n2024-01-15T08:30:00.000Z\nIPv4: 192.168.1.1 | IPv6: ::1`, clear: '' }; // ── State ────────────────────────────────────────────────── const panels = [ { tokenizer: null, modelId: null, view: 'text' }, { tokenizer: null, modelId: null, view: 'text' }, ]; let tokenizerCache = {}; let panel1Visible = true; let debounceTimer = null; let mobileActiveTab = 'input'; // ── DOM References ───────────────────────────────────────── const $overlay = document.getElementById('loading-overlay'); const $loadTitle = document.getElementById('loading-title'); const $loadSub = document.getElementById('loading-sub'); const $loadBar = document.getElementById('loading-bar'); const $loadFile = document.getElementById('loading-file'); const $input = document.getElementById('input-area'); const $charCount = document.getElementById('char-count'); const $toast = document.getElementById('toast'); const $mainGrid = document.getElementById('main-grid'); const $panelToggle = document.getElementById('panel-toggle'); const $themeToggle = document.getElementById('theme-toggle'); const $inputPanel = document.getElementById('input-panel'); const $panel0 = document.getElementById('panel-0'); const $panel1 = document.getElementById('panel-1'); const $mobileTabs = document.querySelectorAll('.mobile-tab'); // ── Utilities ────────────────────────────────────────────── function showOverlay(title, sub) { $loadTitle.textContent = title; $loadSub.textContent = sub; $loadBar.style.width = '0%'; $loadFile.textContent = ''; $overlay.classList.remove('hidden'); } function hideOverlay() { $overlay.classList.add('hidden'); } function showToast(msg, duration = 5000) { $toast.textContent = msg; $toast.classList.add('show'); setTimeout(() => $toast.classList.remove('show'), duration); } function setStats(idx, tokens, text) { const chars = text.length; const words = text.trim() ? text.trim().split(/\s+/).length : 0; const ratio = tokens > 0 && chars > 0 ? (chars / tokens).toFixed(2) : 'β€”'; document.getElementById(`stat-tokens-${idx}`).textContent = tokens > 0 ? tokens.toLocaleString() : 'β€”'; document.getElementById(`stat-chars-${idx}`).textContent = chars > 0 ? chars.toLocaleString() : 'β€”'; document.getElementById(`stat-words-${idx}`).textContent = words > 0 ? words.toLocaleString() : 'β€”'; document.getElementById(`stat-ratio-${idx}`).textContent = ratio; ['tokens','chars','words','ratio'].forEach(k => { const el = document.getElementById(`sc-${k}-${idx}`); el.classList.remove('highlight'); void el.offsetWidth; el.classList.add('highlight'); }); } function updateModelIndicator(idx, modelId) { const preset = MODELS.find(m => m.id === modelId); const color = preset ? preset.color : '#7899c0'; const name = modelId ? modelId.split('/').pop() : 'no model'; const label = idx === 0 ? 'A' : 'B'; document.getElementById(`model-dot-${idx}`).style.background = color; document.getElementById(`model-dot-${idx}`).style.boxShadow = `0 0 6px ${color}`; document.getElementById(`model-label-${idx}`).textContent = `${label}: ${name}`; document.getElementById(`stat-model-${idx}`).textContent = preset ? `${preset.org} Β· ${preset.type} Β· ${preset.vocab} vocab` : modelId || 'no model loaded'; } // ── Decode raw token string for display ─────────────────── function decodeTokenString(raw) { if (!raw) return ''; let s = raw.replace(/^Δ /, ' ').replace(/Δ /g, ' '); s = s.replace(/^▁/, ' ').replace(/▁/g, ' '); s = s.replace(/Ċ/g, '\n'); s = s.replace(/\r/g, ''); s = s.replace(/<0x([0-9A-Fa-f]{2})>/g, (_, hex) => { const code = parseInt(hex, 16); return code < 128 ? String.fromCharCode(code) : `[0x${hex}]`; }); return s; } // ── Mobile Tab Handling ──────────────────────────────────── function isMobile() { return window.innerWidth <= 600; } function applyMobileTab(tabId) { mobileActiveTab = tabId; $mobileTabs.forEach(t => t.classList.toggle('active', t.dataset.tab === tabId)); // Remove mobile-active from all panels $inputPanel.classList.remove('mobile-active'); $panel0.classList.remove('mobile-active'); $panel1.classList.remove('mobile-active'); // Add to the target if (tabId === 'input') $inputPanel.classList.add('mobile-active'); if (tabId === 'panel-0') $panel0.classList.add('mobile-active'); if (tabId === 'panel-1') $panel1.classList.add('mobile-active'); } $mobileTabs.forEach(tab => { tab.addEventListener('click', () => { applyMobileTab(tab.dataset.tab); }); }); // Handle resize: reset display properties when switching between mobile/desktop function handleResize() { if (!isMobile()) { // Desktop/tablet: remove mobile-active, reset display for all panels $inputPanel.classList.remove('mobile-active'); $panel0.classList.remove('mobile-active'); $panel1.classList.remove('mobile-active'); $inputPanel.style.display = ''; $panel0.style.display = ''; $panel1.style.display = panel1Visible ? '' : 'none'; } else { // Mobile: apply mobile tab logic $inputPanel.style.display = ''; $panel0.style.display = ''; $panel1.style.display = ''; applyMobileTab(mobileActiveTab); } updateMobileTabBState(); } function updateMobileTabBState() { const $tabB = document.querySelector('.mobile-tab[data-tab="panel-1"]'); if ($tabB) { if (panel1Visible) { $tabB.classList.remove('tab-disabled'); } else { $tabB.classList.add('tab-disabled'); // If currently on tab B, switch away if (mobileActiveTab === 'panel-1') { applyMobileTab('panel-0'); } } } } window.addEventListener('resize', handleResize); // ── Tokenize for a specific panel ───────────────────────── async function tokenizeForPanel(idx, text) { const p = panels[idx]; const $display = document.getElementById(`token-display-${idx}`); const $placeholder = document.getElementById(`placeholder-${idx}`); if (!p.tokenizer || !text.trim()) { const prevView = $display.querySelector('.token-view-container'); if (prevView) prevView.remove(); if ($placeholder) $placeholder.style.display = 'flex'; setStats(idx, 0, text); return; } try { if ($placeholder) $placeholder.style.display = 'none'; const encoded = await p.tokenizer(text, { add_special_tokens: false }); const ids = Array.from(encoded.input_ids.data); let rawTokens; try { rawTokens = p.tokenizer.model.convert_ids_to_tokens(ids); } catch { rawTokens = await Promise.all(ids.map(id => p.tokenizer.decode([id], { skip_special_tokens: false }))); } const tokens = ids.map((id, i) => ({ id, raw: rawTokens[i] || '', display: decodeTokenString(rawTokens[i] || ''), })); setStats(idx, tokens.length, text); renderView(idx, tokens); } catch (err) { console.error('Tokenization error:', err); showToast(`Panel ${idx === 0 ? 'A' : 'B'} error: ${err.message}`); } } // ── Render Views ─────────────────────────────────────────── function renderView(idx, tokens) { const view = panels[idx].view; if (view === 'text') renderTextView(idx, tokens); else if (view === 'ids') renderIdView(idx, tokens); else if (view === 'list') renderListView(idx, tokens); } function renderTextView(idx, tokens) { const PALETTE = getPalette(); const $display = document.getElementById(`token-display-${idx}`); const container = document.createElement('div'); container.className = 'token-text-view token-view-container fade-in'; tokens.forEach((tok, i) => { const c = PALETTE[i % PALETTE.length]; const span = document.createElement('span'); span.className = 'tok'; span.style.background = c.bg; span.style.color = c.text; span.style.borderBottom = `2px solid ${c.border}`; const disp = tok.display; if (disp === ' ') span.innerHTML = '&nbsp;'; else if (disp === '\n') span.innerHTML = '↡<br>'; else if (disp === '\t') span.innerHTML = 'β†’&nbsp;&nbsp;&nbsp;'; else span.textContent = disp; const tip = document.createElement('div'); tip.className = 'tok-tooltip'; const rawEsc = tok.raw.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;'); tip.innerHTML = `<span class="tok-tooltip-id">#${tok.id}</span> Β· <span class="tok-tooltip-text">${rawEsc || '(empty)'}</span>`; span.appendChild(tip); container.appendChild(span); }); const prevView = $display.querySelector('.token-view-container'); if (prevView) prevView.remove(); document.getElementById(`placeholder-${idx}`).style.display = 'none'; $display.appendChild(container); } function renderIdView(idx, tokens) { const PALETTE = getPalette(); const $display = document.getElementById(`token-display-${idx}`); const container = document.createElement('div'); container.className = 'token-id-view token-view-container fade-in'; tokens.forEach((tok, i) => { const c = PALETTE[i % PALETTE.length]; const card = document.createElement('div'); card.className = 'tok-id-card'; card.style.background = c.bg; card.style.borderColor = c.border; card.title = `Raw: ${tok.raw}`; const top = document.createElement('div'); top.className = 'tok-id-top'; top.style.color = c.text; top.textContent = tok.id; const bot = document.createElement('div'); bot.className = 'tok-id-bottom'; bot.textContent = tok.display.slice(0, 8).replace(/\n/g,'↡').replace(/\t/g,'β†’') || '…'; card.appendChild(top); card.appendChild(bot); container.appendChild(card); }); const prevView = $display.querySelector('.token-view-container'); if (prevView) prevView.remove(); document.getElementById(`placeholder-${idx}`).style.display = 'none'; $display.appendChild(container); } function renderListView(idx, tokens) { const PALETTE = getPalette(); const $display = document.getElementById(`token-display-${idx}`); const container = document.createElement('div'); container.className = 'token-split-view token-view-container fade-in'; tokens.forEach((tok, i) => { const c = PALETTE[i % PALETTE.length]; const row = document.createElement('div'); row.className = 'tok-split-row'; row.style.background = c.bg; row.style.borderColor = c.border; const idxEl = document.createElement('div'); idxEl.className = 'tok-split-idx'; idxEl.textContent = i; const textEl = document.createElement('div'); textEl.className = 'tok-split-text'; textEl.style.color = c.text; textEl.textContent = tok.display.replace(/\n/g,'↡').replace(/\t/g,'β†’') || '(empty)'; const idEl = document.createElement('div'); idEl.className = 'tok-split-id'; idEl.textContent = tok.id; row.appendChild(idxEl); row.appendChild(textEl); row.appendChild(idEl); container.appendChild(row); }); const prevView = $display.querySelector('.token-view-container'); if (prevView) prevView.remove(); document.getElementById(`placeholder-${idx}`).style.display = 'none'; $display.appendChild(container); } // ── Load Tokenizer ───────────────────────────────────────── async function loadModel(idx, modelId) { if (tokenizerCache[modelId]) { panels[idx].tokenizer = tokenizerCache[modelId]; panels[idx].modelId = modelId; updateModelIndicator(idx, modelId); await runTokenize(); return; } const displayName = modelId.split('/').pop(); const label = idx === 0 ? 'A' : 'B'; showOverlay( `Loading ${label}: ${displayName}`, `Fetching tokenizer files from Hugging Face Hub.\nCached in IndexedDB after first download.` ); let lastProgress = 0; try { const tokenizer = await AutoTokenizer.from_pretrained(modelId, { progress_callback: (info) => { if (info.status === 'downloading') { const pct = info.total ? Math.round((info.loaded / info.total) * 100) : lastProgress; $loadBar.style.width = pct + '%'; $loadFile.textContent = info.file || ''; lastProgress = pct; } else if (info.status === 'done') { $loadBar.style.width = '100%'; } } }); tokenizerCache[modelId] = tokenizer; panels[idx].tokenizer = tokenizer; panels[idx].modelId = modelId; updateModelIndicator(idx, modelId); hideOverlay(); await runTokenize(); } catch (err) { hideOverlay(); console.error('Failed to load tokenizer:', err); showToast(`Failed to load "${modelId}": ${err.message}`, 8000); } } // ── Build Dropdown Menus ─────────────────────────────────── function buildDropdowns() { [0, 1].forEach(idx => { const $menu = document.getElementById(`dropdown-menu-${idx}`); $menu.innerHTML = ''; MODELS.forEach(m => { const item = document.createElement('div'); item.className = 'dropdown-item'; item.innerHTML = ` <span class="dropdown-item-dot" style="background:${m.color}"></span> <span class="dropdown-item-name">${m.name}</span> <span class="dropdown-item-detail">${m.org} Β· ${m.type}</span> `; item.addEventListener('click', () => { const $input = document.getElementById(`search-input-${idx}`); $input.value = m.id; $menu.classList.remove('open'); loadModel(idx, m.id); }); $menu.appendChild(item); }); }); } // ── Dropdown toggle ──────────────────────────────────────── [0, 1].forEach(idx => { const $btn = document.getElementById(`dropdown-btn-${idx}`); const $menu = document.getElementById(`dropdown-menu-${idx}`); $btn.addEventListener('click', (e) => { e.stopPropagation(); const otherIdx = 1 - idx; document.getElementById(`dropdown-menu-${otherIdx}`).classList.remove('open'); $menu.classList.toggle('open'); }); }); document.addEventListener('click', () => { document.querySelectorAll('.dropdown-menu').forEach(m => m.classList.remove('open')); }); document.querySelectorAll('.dropdown-menu').forEach(m => { m.addEventListener('click', e => e.stopPropagation()); }); // ── Load buttons ─────────────────────────────────────────── [0, 1].forEach(idx => { const $btn = document.getElementById(`load-btn-${idx}`); const $input = document.getElementById(`search-input-${idx}`); function doLoad() { const id = $input.value.trim(); if (!id) { showToast('Please enter a model ID'); return; } loadModel(idx, id); } $btn.addEventListener('click', doLoad); $input.addEventListener('keydown', e => { if (e.key === 'Enter') doLoad(); }); }); // ── View Toggles ─────────────────────────────────────────── document.querySelectorAll('.toggle-btn').forEach(btn => { btn.addEventListener('click', () => { const panelIdx = parseInt(btn.dataset.panel); const group = document.getElementById(`toggle-group-${panelIdx}`); group.querySelectorAll('.toggle-btn').forEach(b => b.classList.remove('active')); btn.classList.add('active'); panels[panelIdx].view = btn.dataset.view; runTokenize(); }); }); // ── Input Handling ───────────────────────────────────────── async function runTokenize() { const text = $input.value; $charCount.textContent = text.length; await Promise.all([ tokenizeForPanel(0, text), panel1Visible ? tokenizeForPanel(1, text) : Promise.resolve() ]); } $input.addEventListener('input', () => { $charCount.textContent = $input.value.length; clearTimeout(debounceTimer); debounceTimer = setTimeout(runTokenize, 280); }); // ── Sample Buttons ───────────────────────────────────────── document.querySelectorAll('.sample-btn').forEach(btn => { btn.addEventListener('click', () => { $input.value = SAMPLES[btn.dataset.sample] ?? ''; $input.focus(); runTokenize(); }); }); // ── Panel Toggle ─────────────────────────────────────────── $panelToggle.addEventListener('click', () => { panel1Visible = !panel1Visible; $panelToggle.classList.toggle('active', panel1Visible); if (!isMobile()) { $panel1.style.display = panel1Visible ? '' : 'none'; } const $searchGroup1 = document.getElementById('search-group-1'); $searchGroup1.style.display = panel1Visible ? '' : 'none'; if (panel1Visible) { $mainGrid.classList.remove('single-panel'); } else { $mainGrid.classList.add('single-panel'); } updateMobileTabBState(); runTokenize(); }); // ── Theme Toggle ─────────────────────────────────────────── const $iconSun = document.getElementById('theme-icon-sun'); const $iconMoon = document.getElementById('theme-icon-moon'); function setTheme(theme) { document.documentElement.dataset.theme = theme; if (theme === 'light') { $iconSun.style.display = 'none'; $iconMoon.style.display = 'block'; } else { $iconSun.style.display = 'block'; $iconMoon.style.display = 'none'; } runTokenize(); } $themeToggle.addEventListener('click', () => { const current = document.documentElement.dataset.theme; setTheme(current === 'dark' ? 'light' : 'dark'); }); // ── Init ─────────────────────────────────────────────────── buildDropdowns(); $overlay.classList.add('hidden'); $input.value = ''; setTheme('dark'); // Set initial mobile state handleResize(); document.getElementById('search-input-0').value = MODELS[0].id; loadModel(0, MODELS[0].id); </script> </body> </html>
<!DOCTYPE html>
<html lang="en" data-theme="dark">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>TokenLens β€” LLM Tokenizer Playground</title>
<meta name="description" content="Visualize how large language models tokenize text. Powered by Transformers.js, runs entirely in your browser." />
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:opsz,wght@12..96,300;12..96,400;12..96,500;12..96,600;12..96,700;12..96,800&family=JetBrains+Mono:wght@300;400;500;700&family=DM+Sans:wght@300;400;500&display=swap" rel="stylesheet" />
<style>
/* ─── Design Tokens ─────────────────────────────────── */
:root {
--bg: #060b14;
--bg2: #0b1220;
--bg3: #101828;
--bg4: #162035;
--border: #1a2d4a;
--border2: #243d60;
--glow: #1f3d6e;
--text: #dce8f8;
--text2: #7899c0;
--text3: #3d5a80;
--accent: #4d9ef5;
--accent2: #8b6af5;
--green: #34d89a;
--amber: #f5a623;
--red: #f55577;
}
[data-theme="light"] {
--bg: #eef1f8;
--bg2: #e2e7f2;
--bg3: #d3dae8;
--bg4: #c2cce0;
--border: #b8c4d8;
--border2: #a0aec8;
--glow: #c0d0e8;
--text: #1a2236;
--text2: #5a6888;
--text3: #8898b4;
--accent: #2878e0;
--accent2: #6838d8;
--green: #18a060;
--amber: #c88010;
--red: #d83858;
}
/* ─── Reset ─────────────────────────────────────────── */
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
html { height: 100%; overflow: hidden; }
body {
background: var(--bg);
color: var(--text);
font-family: 'DM Sans', sans-serif;
height: 100%;
overflow: hidden;
}
/* ─── Background FX ─────────────────────────────────── */
.bg-gradient {
position: fixed; inset: 0; pointer-events: none; z-index: 0;
background:
radial-gradient(ellipse 80% 50% at 20% 10%, rgba(77,158,245,.06) 0%, transparent 70%),
radial-gradient(ellipse 60% 40% at 80% 90%, rgba(139,106,245,.05) 0%, transparent 60%),
radial-gradient(ellipse 40% 30% at 60% 50%, rgba(52,216,154,.03) 0%, transparent 60%);
}
[data-theme="light"] .bg-gradient {
background:
radial-gradient(ellipse 80% 50% at 20% 10%, rgba(40,120,224,.05) 0%, transparent 70%),
radial-gradient(ellipse 60% 40% at 80% 90%, rgba(104,56,216,.04) 0%, transparent 60%),
radial-gradient(ellipse 40% 30% at 60% 50%, rgba(24,160,96,.02) 0%, transparent 60%);
}
.dot-grid {
position: fixed; inset: 0; pointer-events: none; z-index: 0;
background-image: radial-gradient(circle, rgba(77,158,245,.12) 1px, transparent 1px);
background-size: 36px 36px;
mask-image: radial-gradient(ellipse 100% 100% at 50% 50%, black 30%, transparent 80%);
}
[data-theme="light"] .dot-grid {
background-image: radial-gradient(circle, rgba(40,120,224,.06) 1px, transparent 1px);
}
/* ─── Layout ─────────────────────────────────────────── */
#app {
position: relative; z-index: 1;
display: flex; flex-direction: column;
height: 100vh; overflow: hidden;
}
/* ─── Header ─────────────────────────────────────────── */
header {
display: flex; align-items: center;
padding: 0 20px; height: 56px;
border-bottom: 1px solid var(--border);
background: rgba(6,11,20,.85);
backdrop-filter: blur(20px);
flex-shrink: 0; z-index: 100; gap: 12px;
}
[data-theme="light"] header { background: rgba(238,241,248,.92); }
.logo {
display: flex; align-items: center; gap: 8px;
text-decoration: none; color: var(--text); flex-shrink: 0;
}
.logo-hex {
width: 30px; height: 30px;