Download tests/test_tokenize.py from Merck/TEDDY: direct link, hf CLI and curl.
- Browser
- Download file 5.19 kB
-
https://huggingface.co/Merck/TEDDY/resolve/main/tests/test_tokenize.py
- Command line
-
hf download hf://Merck/TEDDY/tests/test_tokenize.py
-
curl -L -o test_tokenize.py https://huggingface.co/Merck/TEDDY/resolve/main/tests/test_tokenize.py
5.19 kB
| import hashlib | |
| from pathlib import Path | |
| import anndata as ad | |
| import numpy as np | |
| import pytest | |
| import torch | |
| from teddy.data_processing.tokenization.tokenization import ( | |
| _bin_values, | |
| _build_batch_tensors, | |
| _check_genes_in_tokenizer, | |
| _prepare_tokenizer_args, | |
| _rank_continuous, | |
| tokenize, | |
| ) | |
| ############################################################################## | |
| # Unit Tests: Helpers | |
| ############################################################################## | |
| def test_bin_values_no_sorting(): | |
| """ | |
| Test _bin_values with no_sorting=True on a simple array. | |
| """ | |
| vals_list = [np.array([0, 1, 2, 3], dtype=float)] | |
| class TokenArgs: | |
| include_zero_genes = True | |
| bins = 2 | |
| token_args = TokenArgs() | |
| binned = _bin_values(vals_list, token_args, no_sorting=True) | |
| assert len(binned) == 1 | |
| # The result is a single torch tensor with 0 => 0, others => bucket IDs (>=1). | |
| # For example, the min->max range is [0..3], so edges ~ [0, 1.5, 3]. | |
| # Then bucketize: [0 => bin0, 1 => bin1, 2 => bin2, 3 => bin2], but we clamp min=1 for non-zero => final might be [0,1,2,2]. | |
| # Adjust as your code's logic expects. Just do a sanity check: | |
| result = binned[0] | |
| expected = np.array([1, 1, 2, 2], dtype=float) | |
| assert (result.numpy() == expected).all(), f"Got {result.numpy()}, expected {expected}" | |
| def test_bin_values_with_sorting(): | |
| """ | |
| Test _bin_values with no_sorting=False (the 'positional chunk' approach). | |
| """ | |
| vals_list = [np.array([5, 4, 3, 0], dtype=float)] | |
| class TokenArgs: | |
| include_zero_genes = True | |
| bins = 2 | |
| token_args = TokenArgs() | |
| binned = _bin_values(vals_list, token_args, no_sorting=False) | |
| assert len(binned) == 1 | |
| # The 'positional chunk' logic is tested. We won't replicate the entire math here; | |
| # just confirm the shape and that there's a zero-bin appended for the zero entry. | |
| result = binned[0] | |
| assert len(result) == 4, f"Expected 4 bins in the result, got {len(result)}" | |
| def test_rank_continuous_normal(): | |
| """ | |
| Should produce a descending linear scale from ~-1 to +1 across the entire array. | |
| """ | |
| arr = np.array([3, 2, 1, 0], dtype=float) | |
| class TokenArgs: | |
| pass | |
| token_args = TokenArgs() | |
| ranked = _rank_continuous(arr, token_args) | |
| # Should be a descending scale from near -1 => +1 (since we flip). | |
| # len=4 => torch.linspace(-1,1,steps=4) => [-1, -0.3333, +0.3333, +1], flipped => [+1, +0.3333, -0.3333, -1]. | |
| # Just check shape, min, max, monotonic. Exact values might differ slightly if your code does micro offsets. | |
| assert ranked.shape[0] == 4 | |
| assert ranked[0] > ranked[1] # Descending | |
| assert torch.isclose(ranked.min(), torch.tensor(-1.0), atol=1e-5) | |
| def test_prepare_tokenizer_args_dict(): | |
| """ | |
| Test that a dict tokenization_args is converted to TokenizationArgs object properly, | |
| and random seeds are set if gene_seed is not None. | |
| """ | |
| args_dict = { | |
| "load_dir": "/mock/load", | |
| "save_dir": "/mock/save", | |
| "gene_seed": 42, | |
| "tokenizer_name_or_path": "some/tokenizer", | |
| } | |
| token_args_obj, load_dir, save_dir = _prepare_tokenizer_args(args_dict) | |
| assert load_dir == "/mock/load" | |
| assert save_dir == "/mock/save" | |
| assert token_args_obj.gene_seed == 42 | |
| def test_check_genes_in_tokenizer(): | |
| """ | |
| Test _check_genes_in_tokenizer with a minimal mock GeneTokenizer vocab. | |
| """ | |
| # Mock an AnnData with 4 genes, 2 of which are in vocab | |
| import anndata as ad | |
| from teddy.tokenizer.gene_tokenizer import GeneTokenizer | |
| data = ad.AnnData(X=np.zeros((10, 4))) # 10 cells, 4 genes | |
| data.var["gene_name"] = ["G1", "G2", "G3", "G4"] | |
| # Create a mock tokenizer with vocab = {G2, G3} | |
| class MockGeneTokenizer: | |
| def __init__(self, vocab): | |
| self.vocab = vocab | |
| mock_tokenizer = MockGeneTokenizer({"G2": 1, "G3": 2}) | |
| gene_in_vocab, coding_genes, ratio = _check_genes_in_tokenizer( | |
| data, | |
| gene_id_column="gene_name", | |
| tokenizer=mock_tokenizer | |
| ) | |
| # We expect G2, G3 => 2 matches out of 4 => ratio=0.5 | |
| assert len(gene_in_vocab) == 2 | |
| assert ratio == 0.5 | |
| def test_build_batch_tensors(): | |
| """ | |
| Confirm _build_batch_tensors performs topk or random_genes if specified, and returns expected shapes. | |
| We'll do a simple topk test. | |
| """ | |
| X_batch = torch.tensor([[5, 0, 1], [0, 2, 2]], dtype=torch.float) # 2 cells x 3 genes | |
| token_array = torch.tensor([10, 11, 12]) # "gene IDs" for the 3 columns | |
| class TokenArgs: | |
| max_seq_len = 3 | |
| add_cls = False | |
| random_genes = False | |
| gene_ids, vals, labels_list, decoder_vals = _build_batch_tensors(X_batch, token_array, TokenArgs()) | |
| # topk => for each row => pick best 3 (all, since 3 genes). Sorted => descending | |
| # Row0 => [5,1,0], gene IDs => [10,12,11] | |
| # Row1 => [2,2,0], gene IDs => [11,12,10] (tie but stable - depends on how PyTorch breaks ties) | |
| assert len(gene_ids) == 2 | |
| assert labels_list is None | |
| # Check shapes | |
| assert len(gene_ids[0]) == 3 | |
| assert len(vals[0]) == 3 | |
| assert decoder_vals is None | |