"""Decision Index kit engines (github.com/apolinario/decision-index) over DecisionModel. python -m decision_index pipeline --engine decision_model.engine:DecisionModelEngine \\ --option model= --out runs/ python -m decision_index vision pipeline --dir \\ --engine decision_model.engine:DecisionModelVisionEngine \\ --option model= --out runs/-vision Options: model, revision, device, batching ("padded" or "sequential"), max_length (tokens), warmup (true), and for vision max_pixels (default the board's cap, 1,638,400). A request that does not fit max_length, or a question the model cannot answer, is Unsupported (counted as unanswered by the kit); nothing is truncated. """ from __future__ import annotations import importlib.metadata as md import io import time from decision_index.engines.base import Engine, Unsupported from PIL import Image from .model import ( BOARD_MAX_IMAGE_PIXELS, DEFAULT_MAX_LENGTH, DecisionModel, InputTooLong, UnsupportedQuestion, answer, fit_size, render, ) def _versions() -> dict: out = {} for dist in ("torch", "transformers", "flash-linear-attention", "causal-conv1d"): try: out[dist] = md.version(dist) except md.PackageNotFoundError: out[dist] = None return out class DecisionModelEngine(Engine): name = "decision-model" latency = ("In-process request wall time: rendering, tokenization and the forward passes " "of the request's questions on one GPU, until the answers are on the host; " "excludes loading and warm-up.") def __init__(self, model: str, revision: str | None = None, device: str | None = None, batching: str = "padded", max_length: int = DEFAULT_MAX_LENGTH, warmup: bool = True, max_pixels: int | None = None, **options): super().__init__(**options) t = time.perf_counter() self.model = DecisionModel.from_pretrained( str(model), revision=revision, device=device, batching=batching, max_length=int(max_length), max_image_pixels=max_pixels) self.load_seconds = time.perf_counter() - t self.warmup_seconds = self.model.warmup() if warmup else None self.provenance = {"kind": "decision-model", "model": str(model), "revision": revision, "base_model": self.model.config.get("base_model"), "attention_mode": self.model.attention_mode, "temperature": self.model.temperature, "batching": batching, "max_length": int(max_length), "max_image_pixels": self.model.max_image_pixels} def warmup(self): # the kit's own warm-up: done in __init__ pass def __call__(self, state, questions): try: out = self.model.decide(state, questions) except (InputTooLong, UnsupportedQuestion) as exc: raise Unsupported(str(exc)) from exc return {"model": out["model"], "answers": out["answers"]}, None def synchronize(self): import torch if self.model.device_.type == "cuda": torch.cuda.synchronize(self.model.device_) def runtime(self): import torch dev = self.model.device_ return {"device": torch.cuda.get_device_name(dev) if dev.type == "cuda" else str(dev), **_versions(), "attention_check": self.model.attention_check, "load_seconds": self.load_seconds, "warmup_seconds": self.warmup_seconds} class DecisionModelVisionEngine(DecisionModelEngine): """The vision suite: each row's choice question over its images, rendered as the vision protocol (render style "vision"), images at native size up to max_pixels.""" name = "decision-model-vision" latency = ("In-process wall time of a chunk of rows (forward passes planned by length) " "divided by its rows; excludes loading and warm-up.") def __init__(self, model: str, image_root: str | None = None, max_pixels: int = BOARD_MAX_IMAGE_PIXELS, **options): super().__init__(model, max_pixels=int(max_pixels), **options) self.image_root = image_root def _row(self, payload): from decision_index.vision.images import load images = [Image.open(io.BytesIO(load(ref, self.image_root))) for ref in payload.get("images") or ()] for im in images: im.load() rendered = {qid: render(payload.get("state"), q, images, "vision") for qid, q in payload["questions"].items()} pixels = [w * h for w, h in (fit_size(im.width, im.height, self.model.max_image_pixels, self.model.min_image_pixels) for im in images)] return rendered, pixels def _answers(self, payload, rendered, probs): answers = {qid: answer(payload["questions"][qid], keys, p) for (qid, (_, keys)), p in zip(rendered.items(), probs, strict=True)} return {"model": self.model.name, "answers": answers} def batch(self, payloads): out: list = [None] * len(payloads) rows = {} for i, p in enumerate(payloads): try: rows[i] = self._row(p) except UnsupportedQuestion as exc: out[i] = Unsupported(str(exc)) except Exception as exc: # recorded for this row only out[i] = exc flat = [(i, d) for i, (rendered, _) in rows.items() for d, _ in rendered.values()] try: probs = self.model.predict([d for _, d in flat]) k = 0 for i, (rendered, pixels) in rows.items(): n = len(rendered) out[i] = (self._answers(payloads[i], rendered, probs[k:k + n]), {"image_pixels": pixels}) k += n except Exception: # isolate the row that fails; the others still get answers for i, (rendered, pixels) in rows.items(): try: probs = self.model.predict([d for d, _ in rendered.values()]) out[i] = (self._answers(payloads[i], rendered, probs), {"image_pixels": pixels}) except InputTooLong as exc: out[i] = Unsupported(str(exc)) except Exception as exc: out[i] = exc return out def __call__(self, state, questions, images=(), prompt=None): (r,) = self.batch([{"state": state, "questions": questions, "images": images}]) if isinstance(r, Exception): raise r return r