File size: 7,277 Bytes
2b5c0a6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
"""HTTP server over DecisionModel: the model is loaded and warmed up once at startup (kernels
compiled and tuned), then requests run one at a time on it.

    python -m decision_model.server --model <repo or directory> [--port 8000]

POST /v1/systemone   {"model", "state", "questions", "images"?}: the Decision Index text
                     protocol (the kit's `http` engine), rendered as the text engine renders.
POST /v1/decisions   the same body: the Decision Index vision protocol (the kit's `decide`
                     engine with protocol=decisions), rendered as the vision engine renders.
GET  /health, /v1/models

Answers: {"model", "answers": {id: {"type": "choice", "choice", "probabilities"} | {"type":
"noul", "noul"} | {"type": "score", "score", "probabilities"}}, "usage": {"input_tokens",
"output_tokens": 0}}. Capacity refusals use the kit's markers (HTTP 413 "maximum context
length", 422 "options per choice"), so the kit records them as unsupported, not as errors.
Images are base64 data URLs (or bare base64); the server never opens a path from a request.
`server-timing` gives prepare (tokenization, images) and forward (GPU) milliseconds.
A bearer token is required when DECISION_MODEL_API_KEY is set.
"""

from __future__ import annotations

import argparse
import base64
import binascii
import hmac
import os
import threading
import time
from contextlib import asynccontextmanager
from typing import Any

from fastapi import Depends, FastAPI, Header, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ConfigDict, Field

from .model import (
    BOARD_MAX_IMAGE_PIXELS,
    DecisionModel,
    InputTooLong,
    UnsupportedQuestion,
)

MAX_IMAGES = 16
MAX_IMAGE_BYTES = 16_000_000


class Body(BaseModel):
    """A request: the kit's /v1/systemone body, images optional (extra fields ignored)."""

    model_config = ConfigDict(extra="allow")
    model: str = "default"
    state: Any = None
    questions: dict[str, dict[str, Any]] = Field(min_length=1)
    images: list[str] = Field(default_factory=list, max_length=MAX_IMAGES)


class _State:
    model: DecisionModel | None = None
    lock = threading.Lock()
    options: dict[str, Any] = {}
    warmup_seconds: float | None = None


state = _State()


def _image_bytes(value: str) -> bytes:
    data = value.split(",", 1)[1] if value.startswith("data:") else value
    try:
        raw = base64.b64decode(data, validate=True)
    except (binascii.Error, ValueError) as exc:
        raise ValueError("images must be base64 data URLs") from exc
    if len(raw) > MAX_IMAGE_BYTES:
        raise ValueError(f"an image is over {MAX_IMAGE_BYTES} bytes")
    return raw


def create_app(**options):
    """options: model (repo or directory), revision, device, max_image_pixels, batching,
    warmup (bool)."""
    state.options = options

    @asynccontextmanager
    async def lifespan(app):
        from starlette.concurrency import run_in_threadpool

        def load():
            model = DecisionModel.from_pretrained(
                options["model"], revision=options.get("revision"),
                device=options.get("device"), batching=options.get("batching", "padded"),
                max_image_pixels=options.get("max_image_pixels") or BOARD_MAX_IMAGE_PIXELS)
            if options.get("warmup", True):
                state.warmup_seconds = model.warmup()
            return model

        state.model = await run_in_threadpool(load)
        yield
        state.model = None

    app = FastAPI(title="decision-model", lifespan=lifespan)

    def authenticate(authorization: str | None = Header(default=None)) -> None:
        key = os.environ.get("DECISION_MODEL_API_KEY")
        if key and not hmac.compare_digest((authorization or "").encode(),
                                           f"Bearer {key}".encode()):
            raise HTTPException(401, "missing or invalid API key")

    def run(body: Body, style: str) -> JSONResponse:
        model = state.model
        if model is None:
            raise HTTPException(503, "the model is loading")
        try:
            images = [_image_bytes(x) for x in body.images]
        except ValueError as exc:
            raise HTTPException(422, str(exc)) from exc
        with state.lock:  # one request at a time on the GPU; the others wait their turn
            t = time.perf_counter()
            try:
                out = model.decide(body.state, body.questions, images, style)
            except InputTooLong as exc:
                raise HTTPException(413, f"input exceeds the maximum context length of "
                                         f"{model.max_length} tokens: {exc}") from exc
            except UnsupportedQuestion as exc:
                raise HTTPException(422, str(exc)) from exc
            timing = dict(model.last_timing)
            total = time.perf_counter() - t
        header = (f"prepare;dur={1000 * timing['prepare_s']:.1f}, "
                  f"forward;dur={1000 * timing['forward_s']:.1f}, total;dur={1000 * total:.1f}")
        return JSONResponse(out, headers={"server-timing": header})

    @app.post("/v1/systemone", dependencies=[Depends(authenticate)])
    def systemone(body: Body) -> JSONResponse:
        return run(body, "text")

    @app.post("/v1/decisions", dependencies=[Depends(authenticate)])
    def decisions(body: Body) -> JSONResponse:
        return run(body, "vision")

    @app.get("/health")
    def health() -> dict[str, Any]:
        m = state.model
        return {"status": "ready" if m is not None else "loading",
                "model": None if m is None else m.name,
                "warmup_seconds": state.warmup_seconds,
                "attention": None if m is None else m.attention_check}

    @app.get("/v1/models", dependencies=[Depends(authenticate)])
    def models() -> dict[str, Any]:
        m = state.model
        return {"models": [] if m is None else [{"name": m.name}]}

    @app.exception_handler(ValueError)
    async def value_error(request: Request, exc: ValueError) -> JSONResponse:
        return JSONResponse({"detail": str(exc)}, status_code=422)

    return app


def main() -> None:
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
    p.add_argument("--model", default=os.environ.get("DECISION_MODEL"), required=
                   "DECISION_MODEL" not in os.environ, help="Hub repository or directory")
    p.add_argument("--revision", default=os.environ.get("DECISION_MODEL_REVISION"))
    p.add_argument("--device", default=os.environ.get("DECISION_MODEL_DEVICE"))
    p.add_argument("--max-image-pixels", type=int, default=BOARD_MAX_IMAGE_PIXELS)
    p.add_argument("--batching", choices=("padded", "sequential"), default="padded")
    p.add_argument("--no-warmup", action="store_true")
    p.add_argument("--host", default="127.0.0.1")
    p.add_argument("--port", type=int, default=8000)
    a = p.parse_args()
    import uvicorn

    app = create_app(model=a.model, revision=a.revision, device=a.device,
                     max_image_pixels=a.max_image_pixels, batching=a.batching,
                     warmup=not a.no_warmup)
    uvicorn.run(app, host=a.host, port=a.port, workers=1)


if __name__ == "__main__":
    main()