File size: 7,277 Bytes
2b5c0a6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 | """HTTP server over DecisionModel: the model is loaded and warmed up once at startup (kernels
compiled and tuned), then requests run one at a time on it.
python -m decision_model.server --model <repo or directory> [--port 8000]
POST /v1/systemone {"model", "state", "questions", "images"?}: the Decision Index text
protocol (the kit's `http` engine), rendered as the text engine renders.
POST /v1/decisions the same body: the Decision Index vision protocol (the kit's `decide`
engine with protocol=decisions), rendered as the vision engine renders.
GET /health, /v1/models
Answers: {"model", "answers": {id: {"type": "choice", "choice", "probabilities"} | {"type":
"noul", "noul"} | {"type": "score", "score", "probabilities"}}, "usage": {"input_tokens",
"output_tokens": 0}}. Capacity refusals use the kit's markers (HTTP 413 "maximum context
length", 422 "options per choice"), so the kit records them as unsupported, not as errors.
Images are base64 data URLs (or bare base64); the server never opens a path from a request.
`server-timing` gives prepare (tokenization, images) and forward (GPU) milliseconds.
A bearer token is required when DECISION_MODEL_API_KEY is set.
"""
from __future__ import annotations
import argparse
import base64
import binascii
import hmac
import os
import threading
import time
from contextlib import asynccontextmanager
from typing import Any
from fastapi import Depends, FastAPI, Header, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ConfigDict, Field
from .model import (
BOARD_MAX_IMAGE_PIXELS,
DecisionModel,
InputTooLong,
UnsupportedQuestion,
)
MAX_IMAGES = 16
MAX_IMAGE_BYTES = 16_000_000
class Body(BaseModel):
"""A request: the kit's /v1/systemone body, images optional (extra fields ignored)."""
model_config = ConfigDict(extra="allow")
model: str = "default"
state: Any = None
questions: dict[str, dict[str, Any]] = Field(min_length=1)
images: list[str] = Field(default_factory=list, max_length=MAX_IMAGES)
class _State:
model: DecisionModel | None = None
lock = threading.Lock()
options: dict[str, Any] = {}
warmup_seconds: float | None = None
state = _State()
def _image_bytes(value: str) -> bytes:
data = value.split(",", 1)[1] if value.startswith("data:") else value
try:
raw = base64.b64decode(data, validate=True)
except (binascii.Error, ValueError) as exc:
raise ValueError("images must be base64 data URLs") from exc
if len(raw) > MAX_IMAGE_BYTES:
raise ValueError(f"an image is over {MAX_IMAGE_BYTES} bytes")
return raw
def create_app(**options):
"""options: model (repo or directory), revision, device, max_image_pixels, batching,
warmup (bool)."""
state.options = options
@asynccontextmanager
async def lifespan(app):
from starlette.concurrency import run_in_threadpool
def load():
model = DecisionModel.from_pretrained(
options["model"], revision=options.get("revision"),
device=options.get("device"), batching=options.get("batching", "padded"),
max_image_pixels=options.get("max_image_pixels") or BOARD_MAX_IMAGE_PIXELS)
if options.get("warmup", True):
state.warmup_seconds = model.warmup()
return model
state.model = await run_in_threadpool(load)
yield
state.model = None
app = FastAPI(title="decision-model", lifespan=lifespan)
def authenticate(authorization: str | None = Header(default=None)) -> None:
key = os.environ.get("DECISION_MODEL_API_KEY")
if key and not hmac.compare_digest((authorization or "").encode(),
f"Bearer {key}".encode()):
raise HTTPException(401, "missing or invalid API key")
def run(body: Body, style: str) -> JSONResponse:
model = state.model
if model is None:
raise HTTPException(503, "the model is loading")
try:
images = [_image_bytes(x) for x in body.images]
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
with state.lock: # one request at a time on the GPU; the others wait their turn
t = time.perf_counter()
try:
out = model.decide(body.state, body.questions, images, style)
except InputTooLong as exc:
raise HTTPException(413, f"input exceeds the maximum context length of "
f"{model.max_length} tokens: {exc}") from exc
except UnsupportedQuestion as exc:
raise HTTPException(422, str(exc)) from exc
timing = dict(model.last_timing)
total = time.perf_counter() - t
header = (f"prepare;dur={1000 * timing['prepare_s']:.1f}, "
f"forward;dur={1000 * timing['forward_s']:.1f}, total;dur={1000 * total:.1f}")
return JSONResponse(out, headers={"server-timing": header})
@app.post("/v1/systemone", dependencies=[Depends(authenticate)])
def systemone(body: Body) -> JSONResponse:
return run(body, "text")
@app.post("/v1/decisions", dependencies=[Depends(authenticate)])
def decisions(body: Body) -> JSONResponse:
return run(body, "vision")
@app.get("/health")
def health() -> dict[str, Any]:
m = state.model
return {"status": "ready" if m is not None else "loading",
"model": None if m is None else m.name,
"warmup_seconds": state.warmup_seconds,
"attention": None if m is None else m.attention_check}
@app.get("/v1/models", dependencies=[Depends(authenticate)])
def models() -> dict[str, Any]:
m = state.model
return {"models": [] if m is None else [{"name": m.name}]}
@app.exception_handler(ValueError)
async def value_error(request: Request, exc: ValueError) -> JSONResponse:
return JSONResponse({"detail": str(exc)}, status_code=422)
return app
def main() -> None:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
p.add_argument("--model", default=os.environ.get("DECISION_MODEL"), required=
"DECISION_MODEL" not in os.environ, help="Hub repository or directory")
p.add_argument("--revision", default=os.environ.get("DECISION_MODEL_REVISION"))
p.add_argument("--device", default=os.environ.get("DECISION_MODEL_DEVICE"))
p.add_argument("--max-image-pixels", type=int, default=BOARD_MAX_IMAGE_PIXELS)
p.add_argument("--batching", choices=("padded", "sequential"), default="padded")
p.add_argument("--no-warmup", action="store_true")
p.add_argument("--host", default="127.0.0.1")
p.add_argument("--port", type=int, default=8000)
a = p.parse_args()
import uvicorn
app = create_app(model=a.model, revision=a.revision, device=a.device,
max_image_pixels=a.max_image_pixels, batching=a.batching,
warmup=not a.no_warmup)
uvicorn.run(app, host=a.host, port=a.port, workers=1)
if __name__ == "__main__":
main()
|