File size: 4,947 Bytes
238fe38
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
# Copyright 2026 InstaDeep Ltd and CentraleSupélec
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Raw counts -> scFoundation inputs (same steps as in training)."""

from pathlib import Path
from typing import Sequence, Union

import numpy as np
import torch

GENE_LIST_NAME = "gene_list.txt"
CELL_TYPES_NAME = "cell_types.txt"  # training label vocabulary


def load_gene_list(path: Union[str, Path]) -> list[str]:
    """One entry per line (gene symbols in model order, or cell-type labels)."""
    return [g for g in Path(path).read_text().splitlines() if g]


class GeneMapper:
    """Reorders a ``(cells x genes)`` count matrix onto the model's gene list.

    Genes outside the list are dropped, listed genes absent from the input stay
    at zero. ``counts`` may be a numpy array or a scipy sparse matrix.
    """

    def __init__(self, gene_list: Sequence[str]) -> None:
        self.gene_list = list(gene_list)
        self.gene_to_id = {g: i for i, g in enumerate(self.gene_list)}

    def __call__(self, counts, gene_names: Sequence[str]) -> np.ndarray:
        """Dense ``(cells x model genes)`` float32 matrix for ``counts``."""
        if len(gene_names) != counts.shape[1]:
            raise ValueError(f"{len(gene_names)} names for {counts.shape[1]} columns")
        return self.map_columns(counts, self.column_indices(gene_names))

    def column_indices(self, gene_names: Sequence[str]) -> tuple[list[int], list[int]]:
        """``(input columns, model columns)`` of the genes in the model's list."""
        source, target = [], []
        for i, gene in enumerate(gene_names):
            j = self.gene_to_id.get(str(gene))
            if j is not None:
                source.append(i)
                target.append(j)
        if not source:
            raise ValueError("no input gene matches the model gene list (HGNC symbols)")
        return source, target

    def map_columns(self, counts, indices: tuple[list[int], list[int]]) -> np.ndarray:
        """Apply :meth:`column_indices` to a block of cells."""
        source, target = indices
        sub = counts[:, source]
        sub = sub.toarray() if hasattr(sub, "toarray") else np.asarray(sub)
        dense = np.zeros((counts.shape[0], len(self.gene_list)), dtype=np.float32)
        dense[:, target] = sub
        return dense


class ScFoundationPreprocessor(torch.nn.Module):
    """CP10k + log1p, append the two depth tokens, gather the non-zero entries
    and right-pad them to the longest cell of the batch."""

    def __init__(
        self, num_genes: int, pad_token_id: int = 103, target_res_str: str = "t4"
    ) -> None:
        super().__init__()
        self.num_genes = num_genes  # model genes + 2 depth tokens
        self.pad_token_id = pad_token_id
        self.target_res = float(target_res_str[1:])
        self.register_buffer("all_gene_ids", torch.arange(self.num_genes))

    def _gather_and_pad(self, data: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
        batch_size = data.shape[0]
        max_len = int(mask.sum(dim=1).max().item())
        padded_data = torch.full(
            (batch_size, max_len),
            float(self.pad_token_id),
            dtype=data.dtype,
            device=data.device,
        )
        for i in range(batch_size):
            expressed_data = data[i, mask[i]]
            padded_data[i, : len(expressed_data)] = expressed_data
        return padded_data

    def forward(self, gene_expression: torch.Tensor) -> dict[str, torch.Tensor]:
        total_counts = gene_expression.sum(dim=1, keepdim=True) + 1e-9
        log_normalized = torch.log1p(gene_expression / total_counts * 10000)

        log_scaled_counts = torch.log10(total_counts.squeeze(1) * self.target_res)
        log_total_counts = torch.log10(total_counts.squeeze(1))
        special_tokens = torch.stack([log_scaled_counts, log_total_counts], dim=1)
        pretrain_gene_x = torch.cat([log_normalized, special_tokens], dim=1)

        value_mask = pretrain_gene_x > 0
        if not bool(value_mask.any(dim=1).all()):
            raise ValueError("every cell needs at least one non-zero count")
        x = self._gather_and_pad(pretrain_gene_x, value_mask)
        batch_gene_ids = self.all_gene_ids.expand(pretrain_gene_x.shape[0], -1)
        return {
            "x": x,
            "x_padding_mask": x == self.pad_token_id,
            "position_gene_ids": self._gather_and_pad(batch_gene_ids, value_mask),
        }