Spaces:
Running
Running
Download scanner/master.py from FINAL-Bench/ViKernel-Scanner: direct link, hf CLI and curl.
- Browser
- Download file 10.2 kB
-
https://huggingface.co/spaces/FINAL-Bench/ViKernel-Scanner/resolve/main/scanner/master.py
- Command line
-
hf download hf://spaces/FINAL-Bench/ViKernel-Scanner/scanner/master.py
-
curl -L -o master.py https://huggingface.co/spaces/FINAL-Bench/ViKernel-Scanner/resolve/main/scanner/master.py
10.2 kB
| """Master side. Active only on the official Space (VIKERNEL_MASTER=1 and a token that can read the private DB). | |
| What the master controls for every copy of the scanner, including clones: | |
| - the check set (served per request, never shipped in the code) | |
| - the policy (on/off, minimum client version, blocked hosts, notice) | |
| - who scanned: the Hugging Face account and email behind every request | |
| - the official badge (only the master issues badge ids) | |
| """ | |
| import hashlib, hmac, json, os, secrets, threading, time, urllib.error, urllib.request | |
| ENABLED = os.environ.get("VIKERNEL_MASTER") == "1" and bool(os.environ.get("HF_TOKEN")) | |
| DB_REPO = os.environ.get("VIKERNEL_DB_REPO", "FINAL-Bench/vikernel-scanner-db") | |
| BADGE_KEY = (os.environ.get("BADGE_KEY") or secrets.token_hex(16)).encode() | |
| PER_USER_HOUR = int(os.environ.get("MASTER_CHECKS_PER_HOUR", "30")) | |
| REFERENCE_USERS = {u.strip().lower() for u in os.environ.get("REFERENCE_USERS", "SeaWolf-AI,FINAL-Bench").split(",")} | |
| _board = {"t": 0} | |
| if ENABLED: # import once, here: importing it first inside the flush thread races with the main thread | |
| from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download | |
| _ids, _db, _pending, _hits, _badges = {}, {"t": 0}, {}, {}, {} | |
| _lock = threading.Lock() | |
| class Refuse(Exception): | |
| def __init__(self, code, msg): | |
| super().__init__(msg); self.code, self.msg = code, msg | |
| def _api(): | |
| return HfApi(token=os.environ["HF_TOKEN"]) | |
| def _get(path): | |
| return json.load(open(hf_hub_download(DB_REPO, path, repo_type="dataset", token=os.environ["HF_TOKEN"], force_download=True), encoding="utf-8")) | |
| def _load_db(): | |
| if time.time() - _db["t"] > 300: | |
| try: | |
| _db.update(policy=_get("policy.json"), free=_get("checkset/free.json"), t=time.time()) | |
| except Exception: # noqa: BLE001 keep the last good copy; refuse only if there never was one | |
| if "free" not in _db: | |
| raise Refuse(503, "The check database is not available right now.") | |
| return _db | |
| def _identity(token): | |
| """Hugging Face account behind a token: OAuth tokens via userinfo, user access tokens via whoami-v2.""" | |
| if not token: | |
| raise Refuse(401, "Sign in with Hugging Face first.") | |
| key = hashlib.sha256(token.encode()).hexdigest() | |
| c = _ids.get(key) | |
| if c and time.time() - c["t"] < 600: | |
| return c | |
| for url, name_k in (("https://huggingface.co/oauth/userinfo", "preferred_username"), ("https://huggingface.co/api/whoami-v2", "name")): | |
| try: | |
| j = json.loads(urllib.request.urlopen(urllib.request.Request(url, headers={"Authorization": "Bearer " + token}), timeout=15).read()) | |
| except (urllib.error.URLError, ValueError, TimeoutError): | |
| continue | |
| if j.get(name_k): | |
| c = {"user": j[name_k], "email": j.get("email"), "verified": bool(j.get("email_verified") or j.get("emailVerified")), "t": time.time()} | |
| _ids[key] = c | |
| if len(_ids) > 5000: | |
| _ids.clear() | |
| return c | |
| raise Refuse(401, "Hugging Face sign-in could not be verified. Sign in again.") | |
| def _ver(v): | |
| try: | |
| return tuple(int(x) for x in str(v).split(".")[:3]) | |
| except ValueError: | |
| return (0,) | |
| def _queue(path, obj): | |
| with _lock: | |
| _pending[path] = json.dumps(obj, ensure_ascii=False, indent=1).encode() | |
| def _flush_loop(): | |
| while True: | |
| time.sleep(60) | |
| with _lock: | |
| ops = dict(_pending); _pending.clear() | |
| if not ops: | |
| continue | |
| try: | |
| _api().create_commit(DB_REPO, [CommitOperationAdd(p, b) for p, b in ops.items()], commit_message="scanner activity", repo_type="dataset") | |
| except Exception: # noqa: BLE001 put them back and try again next round | |
| with _lock: | |
| for p, b in ops.items(): | |
| _pending.setdefault(p, b) | |
| def _record_user(idn, host, version): | |
| path = "users/%s.json" % idn["user"] | |
| now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) | |
| rec = _ids.get("rec:" + idn["user"]) | |
| if rec is None: | |
| try: | |
| rec = _get(path) | |
| except Exception: # noqa: BLE001 | |
| rec = {"user": idn["user"], "first_seen": now, "scans": 0, "hosts": []} | |
| rec.update(email=idn.get("email") or rec.get("email"), email_verified=idn.get("verified"), last_seen=now, client=version) | |
| if host not in rec["hosts"]: | |
| rec["hosts"] = (rec["hosts"] + [host])[-20:] | |
| _ids["rec:" + idn["user"]] = rec | |
| _queue(path, rec) | |
| return rec | |
| def _gate(token, host, version): | |
| db = _load_db(); pol = db["policy"]; idn = _identity(token) | |
| if not pol.get("enabled", True): | |
| raise Refuse(503, (pol.get("notice") or {}).get("en") or "The scanner is paused by the maintainers.") | |
| if host in pol.get("blocked_hosts", []): | |
| raise Refuse(403, "This copy of the scanner is not allowed to use the ViKernel database.") | |
| if _ver(version) < _ver(pol.get("min_client", "0")): | |
| raise Refuse(426, "This scanner copy is out of date. Update it from github.com/final-bench/vikernel-scanner (minimum %s)." % pol.get("min_client")) | |
| now = time.time(); h = [t for t in _hits.get(idn["user"], []) if now - t < 3600] | |
| if len(h) >= PER_USER_HOUR: | |
| raise Refuse(429, "Hourly scan limit reached for this account. Try again later.") | |
| h.append(now); _hits[idn["user"]] = h | |
| return db, pol, idn | |
| def handle(path, token, host, version, body=None): | |
| """Returns a JSON-able dict. Raises Refuse(code, message).""" | |
| if path == "/api/master/board": # public | |
| return board() | |
| if path == "/api/master/catalog": # public: what is checked, without the check text | |
| db = _load_db(); cs = db["free"] | |
| pol = db["policy"] | |
| return {"db_version": cs.get("version"), "controls": len(cs.get("controls", [])), "inventory": pol.get("inventory"), | |
| "checks": [{k: c.get(k) for k in ("id", "domain", "stage", "rule", "title")} for c in cs.get("checks", [])]} | |
| if path == "/api/master/checks": | |
| db, pol, idn = _gate(token, host, version) | |
| rec = _record_user(idn, host, version) | |
| return {"checkset": db["free"], "user": idn["user"], "has_email": bool(rec.get("email")), | |
| "policy": {"notice": pol.get("notice"), "latest_client": pol.get("latest_client"), "db_version": db["free"].get("version")}} | |
| if path == "/api/master/report": | |
| idn = _identity(token) | |
| b = body or {} | |
| bid = secrets.token_hex(6) | |
| day = time.strftime("%Y-%m-%d", time.gmtime()) | |
| ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) # full-second stamp so the board keeps the newest scan, not a random id | |
| rec = {"id": bid, "user": idn["user"], "host": host, "client": version, "date": day, "ts": ts, "model": str(b.get("model", ""))[:120], | |
| "grade": str(b.get("grade", "?"))[:1], "depth": b.get("depth"), "counts": b.get("counts"), "db_version": b.get("db_version"), | |
| "domains": {k: (v or {}).get("grade") for k, v in (b.get("domains") or {}).items()}, | |
| "hit_checks": b.get("hit_checks"), "checks": b.get("checks"), "controls": b.get("controls"), "firewall": b.get("firewall"), | |
| "reference": idn["user"].lower() in REFERENCE_USERS} | |
| rec["sig"] = hmac.new(BADGE_KEY, json.dumps([bid, rec["model"], rec["grade"], day]).encode(), hashlib.sha256).hexdigest()[:24] | |
| _badges[bid] = rec | |
| _queue("scans/%s/%s.json" % (day, bid), rec) | |
| u = _ids.get("rec:" + idn["user"]) | |
| if u: | |
| u["scans"] = int(u.get("scans", 0)) + 1; _queue("users/%s.json" % idn["user"], u) | |
| return {"badge": {"id": bid, "grade": rec["grade"], "url": "https://final-bench-vikernel-scanner.hf.space/badge/%s.svg" % bid}} | |
| raise Refuse(404, "not found") | |
| def flush_now(): | |
| """Write pending records at once (used by batch runs that exit right after).""" | |
| with _lock: | |
| ops = dict(_pending); _pending.clear() | |
| if ops: | |
| _api().create_commit(DB_REPO, [CommitOperationAdd(p, b) for p, b in ops.items()], commit_message="scanner activity", repo_type="dataset") | |
| def board(): | |
| """Public results: the latest scan of each model. No account names or emails, only whether it is a VIDRAFT reference scan.""" | |
| if time.time() - _board["t"] < 300: | |
| return _board["v"] | |
| recs = list(_badges.values()) | |
| try: | |
| have = {r["id"] for r in recs} | |
| files = sorted((f for f in _api().list_repo_files(DB_REPO, repo_type="dataset") if f.startswith("scans/") and f.endswith(".json")), reverse=True)[:400] | |
| for f in files: | |
| bid = f.rsplit("/", 1)[1][:-5] | |
| if bid not in have: | |
| r = _get(f); _badges[bid] = r; recs.append(r); have.add(bid) | |
| except Exception: # noqa: BLE001 | |
| pass | |
| latest = {} | |
| # Newest scan per model wins. Sort by the full timestamp; older records that predate the ts field | |
| # carry only a day, which sorts before any same-day ts, so a newer stamped scan always supersedes them. | |
| for r in sorted(recs, key=lambda r: (r.get("ts") or r.get("date", ""), r.get("id", ""))): | |
| if r.get("model") and r.get("checks"): | |
| latest[r["model"]] = r | |
| keep = ("id", "model", "grade", "depth", "domains", "hit_checks", "checks", "controls", "date", "reference", "db_version", "firewall") | |
| v = {"models": sorted(({k: r.get(k) for k in keep} for r in latest.values()), key=lambda r: ("ABCDF".find(r.get("grade") or "F"), r.get("hit_checks") or 0)), | |
| "scans": len(recs)} | |
| _board.update(v=v, t=time.time()) | |
| return v | |
| def badge(bid): | |
| """Official badge record for an id, or None.""" | |
| if not all(c in "0123456789abcdef" for c in bid) or len(bid) != 12: | |
| return None | |
| if bid in _badges: | |
| return _badges[bid] | |
| try: | |
| for f in _api().list_repo_files(DB_REPO, repo_type="dataset"): | |
| if f.endswith("/%s.json" % bid): | |
| _badges[bid] = _get(f); return _badges[bid] | |
| except Exception: # noqa: BLE001 | |
| return None | |
| return None | |
| if ENABLED: | |
| threading.Thread(target=_flush_loop, daemon=True).start() | |