CellScribe / modeling_cellscribe.py
mgelard's picture
Initial release of CellScribe
238fe38
Raw History Blame Contribute Delete
10.2 kB
# Copyright 2026 InstaDeep Ltd and CentraleSupélec
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""CellScribe: scFoundation cell encoder + Qwen2.5 conditioned through AdaLN-Zero.
counts -> scFoundation -> pooled embedding [B, 3072]
-> Linear + LayerNorm -> conditioning vector [B, 896]
prompt -> Qwen2.5-0.5B-Instruct whose layer norms are modulated by the
conditioning vector -> "The cell type is ..."
Load with ``AutoModel.from_pretrained("InstaDeepAI/CellScribe",
trust_remote_code=True)``.
"""
from typing import Optional, Sequence
import torch
import torch.nn as nn
import transformers
from transformers import PreTrainedModel
from .ada_ln_zero import attach_ada_ln_zero, create_conditional_model_class
from .configuration_cellscribe import CellScribeConfig
from .postprocessing import extract_predicted_cell_type, parse_cell_state_prediction
from .preprocessing import (
CELL_TYPES_NAME,
GENE_LIST_NAME,
GeneMapper,
ScFoundationPreprocessor,
load_gene_list,
)
from .prompts import build_in_context_prompt, build_prompt
from .scfoundation import SCFoundationWrapper
class LinearHead(nn.Module):
def __init__(self, input_size: int, output_size: int) -> None:
super().__init__()
self.network = nn.Sequential(nn.Linear(input_size, output_size))
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.network(x)
class CellScribe(PreTrainedModel):
config_class = CellScribeConfig
# The output embeddings are tied to the input embeddings (transformers 5 maps
# each tied weight to its source, transformers 4 lists the tied weights).
_tied_weights_keys = (
{"llm.lm_head.weight": "llm.model.embed_tokens.weight"}
if int(transformers.__version__.split(".")[0]) >= 5
else ["llm.lm_head.weight"]
)
def __init__(self, config: CellScribeConfig) -> None:
super().__init__(config)
self.scrna_encoder = SCFoundationWrapper(config.scfoundation)
llm_config = transformers.AutoConfig.for_model(**config.llm)
text_embedding_dim = llm_config.hidden_size
self.scrna_projector = LinearHead(
config.scrnaseq_embedding_dim, text_embedding_dim
)
self.conditioning_norm = nn.LayerNorm(
text_embedding_dim, elementwise_affine=False, eps=1e-6
)
base_llm_class = getattr(transformers, llm_config.architectures[0])
conditional_llm = create_conditional_model_class(base_llm_class)(llm_config)
self.llm = attach_ada_ln_zero(conditional_llm, text_embedding_dim)
# Set by from_pretrained (files next to the weights).
self.gene_list: Optional[list[str]] = None
self.cell_types: Optional[list[str]] = None
# Input helpers, kept out of the module tree (and of the state dict).
self._input_helpers: dict = {}
self.post_init()
def _init_weights(self, module: nn.Module) -> None:
"""Every weight comes from the checkpoint: nothing to initialise."""
def get_input_embeddings(self) -> nn.Module:
return self.llm.get_input_embeddings()
def get_output_embeddings(self) -> nn.Module:
return self.llm.get_output_embeddings()
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
model = super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs)
from transformers.utils import cached_file
hub_kwargs = {
k: kwargs[k]
for k in ("cache_dir", "revision", "token", "local_files_only")
if k in kwargs
}
model.gene_list = load_gene_list(
cached_file(pretrained_model_name_or_path, GENE_LIST_NAME, **hub_kwargs)
)
cell_types = cached_file(
pretrained_model_name_or_path,
CELL_TYPES_NAME,
_raise_exceptions_for_missing_entries=False,
**hub_kwargs,
)
model.cell_types = load_gene_list(cell_types) if cell_types else None
return model
# ------------------------------------------------------------ inputs
def preprocess(self, counts, gene_names: Sequence[str]) -> dict:
"""Raw counts ``(cells x genes)`` named by gene symbols -> the
``scrnaseq_preprocessed_input`` of :meth:`forward` / :meth:`generate`.
Genes are mapped onto the model's gene list (``gene_list.txt``): listed
genes absent from the input are set to zero, other genes are ignored.
"""
if self.gene_list is None:
raise ValueError("load the model with from_pretrained (gene list)")
if not self._input_helpers:
cfg = self.config.scfoundation
self._input_helpers["mapper"] = GeneMapper(self.gene_list)
self._input_helpers["preprocessor"] = ScFoundationPreprocessor(
num_genes=len(self.gene_list) + 2, # + 2 sequencing-depth tokens
pad_token_id=cfg["pad_token_id"],
target_res_str=cfg["tgthighres"],
)
preprocessor = self._input_helpers["preprocessor"].to(self.device)
mapped = torch.from_numpy(self._input_helpers["mapper"](counts, gene_names))
return preprocessor(mapped.to(self.device))
@staticmethod
def build_prompt(tokenizer, task: str) -> str:
"""Chat prompt of a task: ``"cell_typing"`` or ``"cell_state"``."""
return build_prompt(tokenizer, task)
@staticmethod
def build_in_context_prompt(tokenizer, example_labels: Sequence[str]):
"""In-context prompt token ids and the token span of each example line."""
return build_in_context_prompt(tokenizer, example_labels)
# ------------------------------------------------------------ outputs
def parse_cell_type(self, text: str) -> str:
"""Cell type of a generated cell-typing answer, read against the training
vocabulary (``cell_types.txt``), as in the paper's evaluation."""
return extract_predicted_cell_type(text, self.cell_types)
@staticmethod
def parse_cell_state(text: str):
"""``(programs, hallmark)`` of a generated cell-state answer: the
``(meta-program, score)`` pairs, strongest first, and the dominant
hallmark."""
return parse_cell_state_prediction(text)
def predictor(self, tokenizer):
"""High-level :class:`CellScribePredictor`: raw counts in, parsed cell
types / cell states out. Puts the model in eval mode."""
from .inference import CellScribePredictor
return CellScribePredictor(model=self, tokenizer=tokenizer)
# ------------------------------------------------------------ model
def _transform_conditioning(self, embeddings: torch.Tensor) -> torch.Tensor:
return self.conditioning_norm(self.scrna_projector(embeddings))
def embed_cells(self, scrnaseq_preprocessed_input: dict) -> torch.Tensor:
"""Preprocessed cells -> conditioning vectors ``[B, hidden_size]``."""
return self._transform_conditioning(
self.scrna_encoder(**scrnaseq_preprocessed_input)
)
def _get_conditioning(
self,
scrnaseq_preprocessed_input: Optional[dict],
conditioning: Optional[torch.Tensor],
) -> torch.Tensor:
if conditioning is not None:
return conditioning
if scrnaseq_preprocessed_input is None:
raise ValueError(
"Pass either 'scrnaseq_preprocessed_input' or 'conditioning'."
)
return self.embed_cells(scrnaseq_preprocessed_input)
def forward(
self,
input_ids: torch.Tensor,
attention_mask: Optional[torch.Tensor] = None,
scrnaseq_preprocessed_input: Optional[dict] = None,
conditioning: Optional[torch.Tensor] = None,
conditioning_default: Optional[torch.Tensor] = None,
**kwargs,
):
"""Teacher-forced pass.
Give either the preprocessed cells or a ``conditioning`` computed by
:meth:`embed_cells`: ``[B, H]`` (one cell per sequence) or ``[B, T, H]``
(one cell per token, for in-context learning; positions past ``T`` use
``conditioning_default``, ``[B, H]``).
"""
conditioning = self._get_conditioning(scrnaseq_preprocessed_input, conditioning)
return self.llm(
input_ids=input_ids,
attention_mask=attention_mask,
conditioning_signal=conditioning,
conditioning_default=conditioning_default,
**kwargs,
)
@torch.no_grad()
def generate( # type: ignore[override]
self,
input_ids: torch.Tensor,
attention_mask: Optional[torch.Tensor] = None,
scrnaseq_preprocessed_input: Optional[dict] = None,
conditioning: Optional[torch.Tensor] = None,
conditioning_default: Optional[torch.Tensor] = None,
**kwargs,
) -> torch.Tensor:
"""Greedy generation; returns only the generated tokens.
``conditioning`` / ``conditioning_default``: as in :meth:`forward`. With a
per-token ``[B, T, H]`` plan covering the prompt, the generated tokens are
conditioned by ``conditioning_default`` (the query cell).
"""
conditioning = self._get_conditioning(scrnaseq_preprocessed_input, conditioning)
kwargs.setdefault("max_new_tokens", self.config.max_new_tokens)
output = self.llm.generate(
input_ids=input_ids,
attention_mask=attention_mask,
conditioning_signal=conditioning,
conditioning_default=conditioning_default,
**kwargs,
)
return output[:, input_ids.shape[1] :]