abdufelsayed's picture
download
raw
6.23 kB
"""Deployment entry point. One bounded GPU call; CPU-only backup endpoints."""
import json
from pathlib import Path
import sys
import modal
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src") if modal.is_local() else "/opt/fatima/src")
from spec_discovery.execution.modal import APP, VOLUME, MAX_SECONDS # noqa: E402
app = modal.App(APP)
results = modal.Volume.from_name(VOLUME, create_if_missing=True)
cache = modal.Volume.from_name("models-cache")
cpu_image = image = None
if modal.is_local():
cpu_image = (
modal.Image.debian_slim(python_version="3.12")
.add_local_dir(
ROOT / "src",
"/opt/fatima/src",
ignore=["**/node_modules/**", "**/__pycache__/**"],
copy=True,
)
.add_local_dir(ROOT / "configs", "/opt/fatima/configs", copy=True)
.env({"PYTHONPATH": "/opt/fatima/src"})
)
config = json.loads((ROOT / "configs/k2.json").read_text())
image = (
modal.Image.from_registry(
"vllm/vllm-openai:v" + config["runtime"]["vllm"],
setup_dockerfile_commands=["RUN ln -sf /usr/bin/python3 /usr/local/bin/python"],
)
.entrypoint([])
.pip_install(
"transformers==" + config["runtime"]["transformers"],
"httpx==0.28.1",
"nodejs-wheel==22.16.0",
)
.add_local_file(
ROOT / "src/spec_discovery/package.json",
"/opt/fatima/src/spec_discovery/package.json",
copy=True,
)
.add_local_file(
ROOT / "src/spec_discovery/package-lock.json",
"/opt/fatima/src/spec_discovery/package-lock.json",
copy=True,
)
.run_commands("npm ci --ignore-scripts --prefix /opt/fatima/src/spec_discovery")
.add_local_dir(
ROOT / "src",
"/opt/fatima/src",
ignore=["**/node_modules/**", "**/__pycache__/**"],
copy=True,
)
.add_local_dir(ROOT / "configs", "/opt/fatima/configs", copy=True)
.env(
{
"PYTHONPATH": "/opt/fatima/src",
"PYTHONUNBUFFERED": "1",
"HF_HOME": "/root/.cache/huggingface",
"VLLM_LOG_STATS_INTERVAL": "5",
}
)
)
def directory(name):
import re
if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_-]*", name):
raise ValueError("Invalid study name")
return Path("/results") / name
@app.function(
image=image, cpu=(1, 2), memory=(2048, 4096), timeout=180, scaledown_window=2, retries=0
)
def check_runtime(data: bytes, checksum: str):
"""CPU-only check of the deployed dependencies and frozen source installation."""
from importlib.metadata import version
import shutil
import tempfile
from spec_discovery.execution.modal import install_inputs
from spec_discovery.artifacts.provenance import frozen_design, verify_source
from spec_discovery.quint.runtime import runtime_preflight, request
from spec_discovery.authoring.examples import LIFT_SOURCE
with tempfile.TemporaryDirectory() as td:
out = Path(td) / "study"
install_inputs(data, checksum, out, Path("/opt/fatima"))
study = frozen_design(out)
verify_source(study)
packages = {name: version(name) for name in ("vllm", "transformers")}
for name, installed in packages.items():
if installed != study["runtime"][name]:
raise RuntimeError("Serving dependency mismatch: " + name)
runtime_preflight()
request(dict(mode="check", source=LIFT_SOURCE))
executable = Path(sys.executable).with_name("vllm")
if not executable.is_file():
raise RuntimeError("Worker cannot locate the vLLM executable")
return dict(
packages=packages,
quint="0.33.0",
vllm=str(executable),
node=shutil.which("node"),
frozen_source="verified",
)
@app.function(
image=image,
gpu="A100-80GB",
cpu=(4, 4),
memory=(32768, 32768),
volumes={"/results": results, "/root/.cache/huggingface": cache},
timeout=MAX_SECONDS,
max_containers=1,
scaledown_window=2,
retries=0,
single_use_containers=True,
)
def run_study(name: str, data: bytes, checksum: str):
from spec_discovery.execution.modal import install_inputs, managed_worker
from spec_discovery.artifacts.storage import write_bytes, write_json
project = Path("/opt/fatima")
out = Path("/tmp/studies") / name
durable = directory(name)
durable.mkdir(parents=True, exist_ok=False)
results.commit()
install_inputs(data, checksum, out, project)
def publish(content, digest):
write_bytes(durable / "snapshot.zip", content)
write_json(durable / "snapshot.json", dict(sha256=digest))
results.commit()
def acknowledged(freeze_hash, receipt_hash):
results.reload()
path = durable / "acknowledged.json"
return path.exists() and json.loads(path.read_text()) == dict(
freeze_sha256=freeze_hash, receipt_sha256=receipt_hash
)
return managed_worker(out, project, publish, acknowledged)
@app.function(
image=cpu_image,
volumes={"/results": results},
cpu=(0.25, 1),
memory=(512, 2048),
timeout=120,
scaledown_window=2,
retries=0,
)
def snapshot(name: str, known: dict):
from spec_discovery.execution.modal import incremental_snapshot
results.reload()
return incremental_snapshot((directory(name) / "snapshot.zip").read_bytes(), known)
@app.function(
image=cpu_image,
volumes={"/results": results},
cpu=(0.125, 1),
memory=(256, 1024),
timeout=60,
scaledown_window=2,
retries=0,
)
def acknowledge(name: str, freeze_hash: str, receipt_hash: str):
from spec_discovery.execution.modal import acknowledge_snapshot
from spec_discovery.artifacts.storage import write_json
results.reload()
root = directory(name)
receipt = acknowledge_snapshot((root / "snapshot.zip").read_bytes(), freeze_hash, receipt_hash)
write_json(root / "acknowledged.json", receipt)
results.commit()

Xet Storage Details

Size:
6.23 kB
·
Xet hash:
a8a7e8c152f7b3ec46ff2bc6d2824c5d6aefa98eee20a4257491a921354fb1e4

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.