# Copyright 2026 InstaDeep Ltd and CentraleSupélec # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """CellScribe: scFoundation cell encoder + Qwen2.5 conditioned through AdaLN-Zero. counts -> scFoundation -> pooled embedding [B, 3072] -> Linear + LayerNorm -> conditioning vector [B, 896] prompt -> Qwen2.5-0.5B-Instruct whose layer norms are modulated by the conditioning vector -> "The cell type is ..." Load with ``AutoModel.from_pretrained("InstaDeepAI/CellScribe", trust_remote_code=True)``. """ from typing import Optional, Sequence import torch import torch.nn as nn import transformers from transformers import PreTrainedModel from .ada_ln_zero import attach_ada_ln_zero, create_conditional_model_class from .configuration_cellscribe import CellScribeConfig from .postprocessing import extract_predicted_cell_type, parse_cell_state_prediction from .preprocessing import ( CELL_TYPES_NAME, GENE_LIST_NAME, GeneMapper, ScFoundationPreprocessor, load_gene_list, ) from .prompts import build_in_context_prompt, build_prompt from .scfoundation import SCFoundationWrapper class LinearHead(nn.Module): def __init__(self, input_size: int, output_size: int) -> None: super().__init__() self.network = nn.Sequential(nn.Linear(input_size, output_size)) def forward(self, x: torch.Tensor) -> torch.Tensor: return self.network(x) class CellScribe(PreTrainedModel): config_class = CellScribeConfig # The output embeddings are tied to the input embeddings (transformers 5 maps # each tied weight to its source, transformers 4 lists the tied weights). _tied_weights_keys = ( {"llm.lm_head.weight": "llm.model.embed_tokens.weight"} if int(transformers.__version__.split(".")[0]) >= 5 else ["llm.lm_head.weight"] ) def __init__(self, config: CellScribeConfig) -> None: super().__init__(config) self.scrna_encoder = SCFoundationWrapper(config.scfoundation) llm_config = transformers.AutoConfig.for_model(**config.llm) text_embedding_dim = llm_config.hidden_size self.scrna_projector = LinearHead( config.scrnaseq_embedding_dim, text_embedding_dim ) self.conditioning_norm = nn.LayerNorm( text_embedding_dim, elementwise_affine=False, eps=1e-6 ) base_llm_class = getattr(transformers, llm_config.architectures[0]) conditional_llm = create_conditional_model_class(base_llm_class)(llm_config) self.llm = attach_ada_ln_zero(conditional_llm, text_embedding_dim) # Set by from_pretrained (files next to the weights). self.gene_list: Optional[list[str]] = None self.cell_types: Optional[list[str]] = None # Input helpers, kept out of the module tree (and of the state dict). self._input_helpers: dict = {} self.post_init() def _init_weights(self, module: nn.Module) -> None: """Every weight comes from the checkpoint: nothing to initialise.""" def get_input_embeddings(self) -> nn.Module: return self.llm.get_input_embeddings() def get_output_embeddings(self) -> nn.Module: return self.llm.get_output_embeddings() @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs): model = super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs) from transformers.utils import cached_file hub_kwargs = { k: kwargs[k] for k in ("cache_dir", "revision", "token", "local_files_only") if k in kwargs } model.gene_list = load_gene_list( cached_file(pretrained_model_name_or_path, GENE_LIST_NAME, **hub_kwargs) ) cell_types = cached_file( pretrained_model_name_or_path, CELL_TYPES_NAME, _raise_exceptions_for_missing_entries=False, **hub_kwargs, ) model.cell_types = load_gene_list(cell_types) if cell_types else None return model # ------------------------------------------------------------ inputs def preprocess(self, counts, gene_names: Sequence[str]) -> dict: """Raw counts ``(cells x genes)`` named by gene symbols -> the ``scrnaseq_preprocessed_input`` of :meth:`forward` / :meth:`generate`. Genes are mapped onto the model's gene list (``gene_list.txt``): listed genes absent from the input are set to zero, other genes are ignored. """ if self.gene_list is None: raise ValueError("load the model with from_pretrained (gene list)") if not self._input_helpers: cfg = self.config.scfoundation self._input_helpers["mapper"] = GeneMapper(self.gene_list) self._input_helpers["preprocessor"] = ScFoundationPreprocessor( num_genes=len(self.gene_list) + 2, # + 2 sequencing-depth tokens pad_token_id=cfg["pad_token_id"], target_res_str=cfg["tgthighres"], ) preprocessor = self._input_helpers["preprocessor"].to(self.device) mapped = torch.from_numpy(self._input_helpers["mapper"](counts, gene_names)) return preprocessor(mapped.to(self.device)) @staticmethod def build_prompt(tokenizer, task: str) -> str: """Chat prompt of a task: ``"cell_typing"`` or ``"cell_state"``.""" return build_prompt(tokenizer, task) @staticmethod def build_in_context_prompt(tokenizer, example_labels: Sequence[str]): """In-context prompt token ids and the token span of each example line.""" return build_in_context_prompt(tokenizer, example_labels) # ------------------------------------------------------------ outputs def parse_cell_type(self, text: str) -> str: """Cell type of a generated cell-typing answer, read against the training vocabulary (``cell_types.txt``), as in the paper's evaluation.""" return extract_predicted_cell_type(text, self.cell_types) @staticmethod def parse_cell_state(text: str): """``(programs, hallmark)`` of a generated cell-state answer: the ``(meta-program, score)`` pairs, strongest first, and the dominant hallmark.""" return parse_cell_state_prediction(text) def predictor(self, tokenizer): """High-level :class:`CellScribePredictor`: raw counts in, parsed cell types / cell states out. Puts the model in eval mode.""" from .inference import CellScribePredictor return CellScribePredictor(model=self, tokenizer=tokenizer) # ------------------------------------------------------------ model def _transform_conditioning(self, embeddings: torch.Tensor) -> torch.Tensor: return self.conditioning_norm(self.scrna_projector(embeddings)) def embed_cells(self, scrnaseq_preprocessed_input: dict) -> torch.Tensor: """Preprocessed cells -> conditioning vectors ``[B, hidden_size]``.""" return self._transform_conditioning( self.scrna_encoder(**scrnaseq_preprocessed_input) ) def _get_conditioning( self, scrnaseq_preprocessed_input: Optional[dict], conditioning: Optional[torch.Tensor], ) -> torch.Tensor: if conditioning is not None: return conditioning if scrnaseq_preprocessed_input is None: raise ValueError( "Pass either 'scrnaseq_preprocessed_input' or 'conditioning'." ) return self.embed_cells(scrnaseq_preprocessed_input) def forward( self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, scrnaseq_preprocessed_input: Optional[dict] = None, conditioning: Optional[torch.Tensor] = None, conditioning_default: Optional[torch.Tensor] = None, **kwargs, ): """Teacher-forced pass. Give either the preprocessed cells or a ``conditioning`` computed by :meth:`embed_cells`: ``[B, H]`` (one cell per sequence) or ``[B, T, H]`` (one cell per token, for in-context learning; positions past ``T`` use ``conditioning_default``, ``[B, H]``). """ conditioning = self._get_conditioning(scrnaseq_preprocessed_input, conditioning) return self.llm( input_ids=input_ids, attention_mask=attention_mask, conditioning_signal=conditioning, conditioning_default=conditioning_default, **kwargs, ) @torch.no_grad() def generate( # type: ignore[override] self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, scrnaseq_preprocessed_input: Optional[dict] = None, conditioning: Optional[torch.Tensor] = None, conditioning_default: Optional[torch.Tensor] = None, **kwargs, ) -> torch.Tensor: """Greedy generation; returns only the generated tokens. ``conditioning`` / ``conditioning_default``: as in :meth:`forward`. With a per-token ``[B, T, H]`` plan covering the prompt, the generated tokens are conditioned by ``conditioning_default`` (the query cell). """ conditioning = self._get_conditioning(scrnaseq_preprocessed_input, conditioning) kwargs.setdefault("max_new_tokens", self.config.max_new_tokens) output = self.llm.generate( input_ids=input_ids, attention_mask=attention_mask, conditioning_signal=conditioning, conditioning_default=conditioning_default, **kwargs, ) return output[:, input_ids.shape[1] :]