File size: 6,824 Bytes
2b5c0a6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
"""Decision Index kit engines (github.com/apolinario/decision-index) over DecisionModel.

    python -m decision_index pipeline --engine decision_model.engine:DecisionModelEngine \\
        --option model=<repo or directory> --out runs/<name>
    python -m decision_index vision pipeline --dir <vision suite> \\
        --engine decision_model.engine:DecisionModelVisionEngine \\
        --option model=<repo or directory> --out runs/<name>-vision

Options: model, revision, device, batching ("padded" or "sequential"), max_length (tokens),
warmup (true), and for vision max_pixels (default the board's cap, 1,638,400). A request
that does not fit max_length, or a question the model cannot answer, is Unsupported (counted
as unanswered by the kit); nothing is truncated.
"""

from __future__ import annotations

import importlib.metadata as md
import io
import time

from decision_index.engines.base import Engine, Unsupported
from PIL import Image

from .model import (
    BOARD_MAX_IMAGE_PIXELS,
    DEFAULT_MAX_LENGTH,
    DecisionModel,
    InputTooLong,
    UnsupportedQuestion,
    answer,
    fit_size,
    render,
)


def _versions() -> dict:
    out = {}
    for dist in ("torch", "transformers", "flash-linear-attention", "causal-conv1d"):
        try:
            out[dist] = md.version(dist)
        except md.PackageNotFoundError:
            out[dist] = None
    return out


class DecisionModelEngine(Engine):
    name = "decision-model"
    latency = ("In-process request wall time: rendering, tokenization and the forward passes "
               "of the request's questions on one GPU, until the answers are on the host; "
               "excludes loading and warm-up.")

    def __init__(self, model: str, revision: str | None = None, device: str | None = None,
                 batching: str = "padded", max_length: int = DEFAULT_MAX_LENGTH,
                 warmup: bool = True, max_pixels: int | None = None, **options):
        super().__init__(**options)
        t = time.perf_counter()
        self.model = DecisionModel.from_pretrained(
            str(model), revision=revision, device=device, batching=batching,
            max_length=int(max_length), max_image_pixels=max_pixels)
        self.load_seconds = time.perf_counter() - t
        self.warmup_seconds = self.model.warmup() if warmup else None
        self.provenance = {"kind": "decision-model", "model": str(model), "revision": revision,
                           "base_model": self.model.config.get("base_model"),
                           "attention_mode": self.model.attention_mode,
                           "temperature": self.model.temperature, "batching": batching,
                           "max_length": int(max_length),
                           "max_image_pixels": self.model.max_image_pixels}

    def warmup(self):  # the kit's own warm-up: done in __init__
        pass

    def __call__(self, state, questions):
        try:
            out = self.model.decide(state, questions)
        except (InputTooLong, UnsupportedQuestion) as exc:
            raise Unsupported(str(exc)) from exc
        return {"model": out["model"], "answers": out["answers"]}, None

    def synchronize(self):
        import torch

        if self.model.device_.type == "cuda":
            torch.cuda.synchronize(self.model.device_)

    def runtime(self):
        import torch

        dev = self.model.device_
        return {"device": torch.cuda.get_device_name(dev) if dev.type == "cuda" else str(dev),
                **_versions(), "attention_check": self.model.attention_check,
                "load_seconds": self.load_seconds, "warmup_seconds": self.warmup_seconds}


class DecisionModelVisionEngine(DecisionModelEngine):
    """The vision suite: each row's choice question over its images, rendered as the vision
    protocol (render style "vision"), images at native size up to max_pixels."""

    name = "decision-model-vision"
    latency = ("In-process wall time of a chunk of rows (forward passes planned by length) "
               "divided by its rows; excludes loading and warm-up.")

    def __init__(self, model: str, image_root: str | None = None,
                 max_pixels: int = BOARD_MAX_IMAGE_PIXELS, **options):
        super().__init__(model, max_pixels=int(max_pixels), **options)
        self.image_root = image_root

    def _row(self, payload):
        from decision_index.vision.images import load

        images = [Image.open(io.BytesIO(load(ref, self.image_root)))
                  for ref in payload.get("images") or ()]
        for im in images:
            im.load()
        rendered = {qid: render(payload.get("state"), q, images, "vision")
                    for qid, q in payload["questions"].items()}
        pixels = [w * h for w, h in (fit_size(im.width, im.height, self.model.max_image_pixels,
                                              self.model.min_image_pixels) for im in images)]
        return rendered, pixels

    def _answers(self, payload, rendered, probs):
        answers = {qid: answer(payload["questions"][qid], keys, p)
                   for (qid, (_, keys)), p in zip(rendered.items(), probs, strict=True)}
        return {"model": self.model.name, "answers": answers}

    def batch(self, payloads):
        out: list = [None] * len(payloads)
        rows = {}
        for i, p in enumerate(payloads):
            try:
                rows[i] = self._row(p)
            except UnsupportedQuestion as exc:
                out[i] = Unsupported(str(exc))
            except Exception as exc:  # recorded for this row only
                out[i] = exc
        flat = [(i, d) for i, (rendered, _) in rows.items() for d, _ in rendered.values()]
        try:
            probs = self.model.predict([d for _, d in flat])
            k = 0
            for i, (rendered, pixels) in rows.items():
                n = len(rendered)
                out[i] = (self._answers(payloads[i], rendered, probs[k:k + n]),
                          {"image_pixels": pixels})
                k += n
        except Exception:  # isolate the row that fails; the others still get answers
            for i, (rendered, pixels) in rows.items():
                try:
                    probs = self.model.predict([d for d, _ in rendered.values()])
                    out[i] = (self._answers(payloads[i], rendered, probs),
                              {"image_pixels": pixels})
                except InputTooLong as exc:
                    out[i] = Unsupported(str(exc))
                except Exception as exc:
                    out[i] = exc
        return out

    def __call__(self, state, questions, images=(), prompt=None):
        (r,) = self.batch([{"state": state, "questions": questions, "images": images}])
        if isinstance(r, Exception):
            raise r
        return r