Download dpacman/data_tasks/split/remap_handpick.py from ChatterjeeLab/DPACMAN: direct link, hf CLI and curl.
- Browser
- Download file 12.5 kB
-
https://huggingface.co/ChatterjeeLab/DPACMAN/resolve/main/dpacman/data_tasks/split/remap_handpick.py
- Command line
-
hf download hf://ChatterjeeLab/DPACMAN/dpacman/data_tasks/split/remap_handpick.py
-
curl -L -o remap_handpick.py https://huggingface.co/ChatterjeeLab/DPACMAN/resolve/main/dpacman/data_tasks/split/remap_handpick.py
12.5 kB
| """ | |
| Not neat, but this is what I did to make exclusive splits. saving here for now. | |
| """ | |
| ## Full pipeline | |
| import pandas as pd | |
| protein_clusters = pd.read_csv("/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/mmseqs/outputs/fimo_hits_only/protein/mmseqs_cluster.tsv", sep="\t", header=None) | |
| protein_clusters.columns=["tr_cluster_rep","tr_cluster_member"] | |
| protein_clusters.head() | |
| dna_clusters = pd.read_csv("/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/mmseqs/outputs/fimo_hits_only/dna_full/mmseqs_cluster.tsv", sep="\t", header=None) | |
| dna_clusters.columns=["dna_cluster_rep","dna_cluster_member"] | |
| dna_clusters.head() | |
| all_data = pd.read_parquet("/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/fimo/post_fimo/fimo_hits_only/remap2022_crm_fimo_output_q_processed_seed0.parquet") | |
| all_data | |
| protein_cluster_map = dict(zip(protein_clusters["tr_cluster_member"],protein_clusters["tr_cluster_rep"])) | |
| dna_cluster_map = dict(zip(dna_clusters["dna_cluster_member"],dna_clusters["dna_cluster_rep"])) | |
| print(len(protein_cluster_map)) | |
| print(len(dna_cluster_map)) | |
| all_data["tr_cluster_rep"] = all_data["tr_seqid"].map(protein_cluster_map) | |
| all_data["dna_cluster_rep"] = all_data["dna_seqid"].map(dna_cluster_map) | |
| print(len(all_data[all_data["tr_cluster_rep"].isna()])) | |
| print(len(all_data[all_data["dna_cluster_rep"].isna()])) | |
| all_data.head() | |
| ### handpick test | |
| handpicked_test_trs = ["trseq23","trseq26","trseq17"] | |
| handpicked_test = all_data.loc[ | |
| all_data["tr_cluster_rep"].isin(handpicked_test_trs) | |
| ].reset_index(drop=True) | |
| off_limits_dna_clusters = handpicked_test["dna_cluster_rep"].unique().tolist() | |
| remaining = all_data.loc[ | |
| (~all_data["tr_cluster_rep"].isin(handpicked_test_trs)) & | |
| (~all_data["dna_cluster_rep"].isin(off_limits_dna_clusters)) | |
| ].reset_index(drop=True) | |
| test_ids = handpicked_test["ID"].unique().tolist() | |
| remaining_ids = remaining["ID"].unique().tolist() | |
| lost_rows = all_data.loc[ | |
| (~all_data["ID"].isin(test_ids)) & | |
| (~all_data["ID"].isin(remaining_ids)) | |
| ] | |
| print(f"Rows in test: {len(handpicked_test)}") | |
| print(f"Rows to be split between train and val: {len(remaining)}") | |
| total_rows = len(handpicked_test) + len(remaining) | |
| print(f"Total rows: {total_rows}. Test percentage: {100*len(handpicked_test)/total_rows:.2f}%") | |
| print(f"Lost rows: {len(lost_rows)}") | |
| ### handpick val | |
| handpicked_val_trs = ["trseq9", "trseq5", "trseq28"] | |
| handpicked_val = remaining.loc[ | |
| remaining["tr_cluster_rep"].isin(handpicked_val_trs) | |
| ].reset_index(drop=True) | |
| off_limits_dna_clusters = handpicked_val["dna_cluster_rep"].unique().tolist() | |
| train_remain = remaining.loc[ | |
| (~remaining["tr_cluster_rep"].isin(handpicked_val_trs)) & | |
| (~remaining["dna_cluster_rep"].isin(off_limits_dna_clusters)) | |
| ].reset_index(drop=True) | |
| val_ids = handpicked_val["ID"].unique().tolist() | |
| train_remain_ids = train_remain["ID"].unique().tolist() | |
| lost_rows = all_data.loc[ | |
| (~all_data["ID"].isin(test_ids)) & | |
| (~all_data["ID"].isin(val_ids)) & | |
| (~all_data["ID"].isin(train_remain_ids)) | |
| ] | |
| print(f"Rows in val: {len(handpicked_val)}") | |
| print(f"Rows left for train: {len(train_remain)}") | |
| total_rows = len(handpicked_val) + len(train_remain) | |
| print(f"Total rows: {total_rows}. Test percentage: {100*len(handpicked_val)/total_rows:.2f}%") | |
| print(f"Lost rows: {len(lost_rows)}") | |
| train_exclusive = all_data.loc[ | |
| all_data["ID"].isin(train_remain_ids) | |
| ].reset_index(drop=True) | |
| val_exclusive = all_data.loc[ | |
| all_data["ID"].isin(val_ids) | |
| ].reset_index(drop=True) | |
| test_exclusive = all_data.loc[ | |
| all_data["ID"].isin(test_ids) | |
| ].reset_index(drop=True) | |
| leaky_test = all_data.loc[ | |
| ~(all_data["ID"].isin(train_exclusive["ID"].tolist())) & | |
| ~(all_data["ID"].isin(val_exclusive["ID"].tolist())) & | |
| ~(all_data["ID"].isin(test_exclusive["ID"].tolist())) | |
| ].reset_index(drop=True) | |
| print(f"Original total: {len(all_data)}") | |
| retained_total = len(train_exclusive)+len(val_exclusive)+len(test_exclusive) | |
| print(f"New, exclusive total: {retained_total}") | |
| print(f"Lost rows: {len(all_data)-retained_total}") | |
| print(f"Length train: {len(train_exclusive)}/{retained_total} ({100*len(train_exclusive)/retained_total:.2f}%)") | |
| print(f"Length val: {len(val_exclusive)}/{retained_total} ({100*len(val_exclusive)/retained_total:.2f}%)") | |
| print(f"Length test: {len(test_exclusive)}/{retained_total} ({100*len(test_exclusive)/retained_total:.2f}%)") | |
| def check_validity(train_exclusive, val_exclusive, test_exclusive): | |
| train_exclusive_ids = set(train_exclusive["ID"].unique().tolist()) | |
| val_exclusive_ids = set(val_exclusive["ID"].unique().tolist()) | |
| test_exclusive_ids = set(test_exclusive["ID"].unique().tolist()) | |
| assert len(train_exclusive_ids.intersection(val_exclusive_ids)) == 0 | |
| assert len(train_exclusive_ids.intersection(test_exclusive_ids)) == 0 | |
| assert len(val_exclusive_ids.intersection(test_exclusive_ids)) == 0 | |
| print(f"Pass! No overlap in IDs") | |
| # Investigate TR intersection. No assertions unless we are explicitly splitting on this. | |
| train_exclusive_tr_seqs = set(train_exclusive["tr_sequence"].unique().tolist()) | |
| val_exclusive_tr_seqs = set(val_exclusive["tr_sequence"].unique().tolist()) | |
| test_exclusive_tr_seqs = set(test_exclusive["tr_sequence"].unique().tolist()) | |
| train_exclusive_tr_reps = set(train_exclusive["tr_cluster_rep"].unique().tolist()) | |
| val_exclusive_tr_reps = set(val_exclusive["tr_cluster_rep"].unique().tolist()) | |
| test_exclusive_tr_reps = set(test_exclusive["tr_cluster_rep"].unique().tolist()) | |
| print(f"Train-Val TR intersection: {len(train_exclusive_tr_seqs.intersection(val_exclusive_tr_seqs))}") | |
| print(f"Train-Test TR intersection: {len(train_exclusive_tr_seqs.intersection(test_exclusive_tr_seqs))}") | |
| print(f"Val-Test TR intersection: {len(val_exclusive_tr_seqs.intersection(test_exclusive_tr_seqs))}") | |
| print(f"Train-Val TR Cluster Rep intersection: {len(train_exclusive_tr_reps.intersection(val_exclusive_tr_reps))}") | |
| print(f"Train-Test TR Cluster Rep intersection: {len(train_exclusive_tr_reps.intersection(test_exclusive_tr_reps))}") | |
| print(f"Val-Test TR Cluster Rep intersection: {len(val_exclusive_tr_reps.intersection(test_exclusive_tr_reps))}") | |
| # Investigate DNA intersection. No assertions unless we are explicitly splitting on this. | |
| train_exclusive_dna_seqs = set(train_exclusive["dna_sequence"].unique().tolist()) | |
| val_exclusive_dna_seqs = set(val_exclusive["dna_sequence"].unique().tolist()) | |
| test_exclusive_dna_seqs = set(test_exclusive["dna_sequence"].unique().tolist()) | |
| train_exclusive_dna_reps = set(train_exclusive["dna_cluster_rep"].unique().tolist()) | |
| val_exclusive_dna_reps = set(val_exclusive["dna_cluster_rep"].unique().tolist()) | |
| test_exclusive_dna_reps = set(test_exclusive["dna_cluster_rep"].unique().tolist()) | |
| print(f"Train-Val DNA intersection: {len(train_exclusive_dna_seqs.intersection(val_exclusive_dna_seqs))}") | |
| print(f"Train-Test DNA intersection: {len(train_exclusive_dna_seqs.intersection(test_exclusive_dna_seqs))}") | |
| print(f"Val-Test DNA intersection: {len(val_exclusive_dna_seqs.intersection(test_exclusive_dna_seqs))}") | |
| print(f"Train-Val DNA Cluster Rep intersection: {len(train_exclusive_dna_reps.intersection(val_exclusive_dna_reps))}") | |
| print(f"Train-Test DNA Cluster Rep intersection: {len(train_exclusive_dna_reps.intersection(test_exclusive_dna_reps))}") | |
| print(f"Val-Test DNA Cluster Rep intersection: {len(val_exclusive_dna_reps.intersection(test_exclusive_dna_reps))}") | |
| def get_reverse_complement(s): | |
| """ | |
| Returns 5' to 3' sequence of the reverse complement | |
| """ | |
| chars = list(s) | |
| recon = [] | |
| rev_map = { | |
| "a": "t", | |
| "c": "g", | |
| "t": "a", | |
| "g": "c", | |
| "A": "T", | |
| "C": "G", | |
| "T": "A", | |
| "G": "C", | |
| "n": "n", | |
| "N": "N", | |
| } | |
| for c in chars: | |
| recon += [rev_map[c]] | |
| recon = "".join(recon) | |
| return recon[::-1] | |
| # now make reverse complements | |
| def augment_rc(df): | |
| """ | |
| Get the reverse complement and add it as a datapoint, effectively doubling the dataset. | |
| Also flip the orientation of the scores | |
| columns = ["ID","dna_sequence","tr_sequence","tr_cluster_rep","dna_cluster_rep", "scores","split"] | |
| """ | |
| df_rc = df.copy(deep=True) | |
| df_rc["dna_sequence"] = df_rc["dna_sequence"].apply( | |
| lambda x: get_reverse_complement(x) | |
| ) | |
| df_rc["ID"] = df_rc["ID"] + "_rc" | |
| df_rc["scores"] = df_rc["scores"].apply(lambda s: ",".join(s.split(",")[::-1])) | |
| final_df = pd.concat([df, df_rc]).reset_index(drop=True) | |
| return final_df | |
| def convert_scores(scores, mode=1): | |
| """ | |
| Two modes: 1 means FIMO peaks get 1. 0 means FIMO peaks get their max score | |
| """ | |
| svec = [int(x) for x in scores.split(",")] | |
| max_score = max(svec) | |
| if mode ==1: | |
| binary_svec = [0 if x<max_score else 1 for x in svec] | |
| assert(svec.count(max_score)==binary_svec.count(1)) | |
| else: | |
| binary_svec = [0 if x<max_score else max_score for x in svec] | |
| assert(svec.count(max_score)==binary_svec.count(max_score)) | |
| binary_svec = ",".join([str(x) for x in binary_svec]) | |
| return binary_svec | |
| check_validity(train_exclusive, val_exclusive, test_exclusive) | |
| train_exclusive = augment_rc(train_exclusive) | |
| val_exclusive = augment_rc(val_exclusive) | |
| test_exclusive = augment_rc(test_exclusive) | |
| leaky_test = augment_rc(leaky_test) | |
| print(f"Added reverse complement sequences to train_exclusive, val_exclusive, and test_exclusive (and leaky test_exclusive)") | |
| check_validity(train_exclusive, val_exclusive, test_exclusive) | |
| total = sum([len(train_exclusive), len(val_exclusive), len(test_exclusive), len(leaky_test)]) | |
| print( | |
| f"Length of train_exclusive dataset: {len(train_exclusive)} ({100*len(train_exclusive)/total:.2f}%)" | |
| ) | |
| print(f"Length of val_exclusive dataset: {len(val_exclusive)} ({100*len(val_exclusive)/total:.2f}%)") | |
| print(f"Length of test_exclusive dataset: {len(test_exclusive)} ({100*len(test_exclusive)/total:.2f}%)") | |
| print(f"Length of leaky_test dataset: {len(leaky_test)} ({100*len(leaky_test)/total:.2f}%)") | |
| print( | |
| f"Total sequences = {total}. Same as edges size*2? {total==len(all_data)*2}" | |
| ) | |
| # since we've added all these new DNA sequences, we do need a new mapping of seq id to dna sequence | |
| all_data = pd.concat([train_exclusive, val_exclusive, test_exclusive, leaky_test]) | |
| all_data["dna_seqid"] = all_data["ID"].str.split("_", n=1, expand=True)[1] | |
| dna_dict = dict(zip(all_data["dna_seqid"], all_data["dna_sequence"])) | |
| assert len(dna_dict) == len(all_data.drop_duplicates(["dna_sequence"])) | |
| # create the output dir | |
| import os | |
| from pathlib import Path | |
| split_out_dir = Path("/home/a03-svincoff/DPACMAN/dpacman/data_files/processed/splits/handpicked_val_test") | |
| os.makedirs(split_out_dir, exist_ok=True) | |
| # add binary_scores to allow other training modes | |
| train_exclusive["binary_sores"] = train_exclusive["scores"].apply(lambda x: convert_scores(x, mode=1)) | |
| val_exclusive["binary_sores"] = val_exclusive["scores"].apply(lambda x: convert_scores(x, mode=1)) | |
| test_exclusive["binary_sores"] = test_exclusive["scores"].apply(lambda x: convert_scores(x, mode=1)) | |
| leaky_test["binary_sores"] = leaky_test["scores"].apply(lambda x: convert_scores(x, mode=1)) | |
| train_exclusive["split"] = ["train"]*len(train_exclusive) | |
| val_exclusive["split"] = ["val"]*len(val_exclusive) | |
| test_exclusive["split"] = ["test"]*len(test_exclusive) | |
| leaky_test["split"] = ["leakytest"]*len(leaky_test) | |
| # slect final cols and save | |
| split_final_cols = ["ID", "dna_sequence", "tr_sequence", "scores", "binary_sores", "split"] | |
| train_exclusive[split_final_cols].to_csv(split_out_dir / "train.csv", index=False) | |
| val_exclusive[split_final_cols].to_csv(split_out_dir / "val.csv", index=False) | |
| test_exclusive[split_final_cols].to_csv(split_out_dir / "test.csv", index=False) | |
| leaky_test[split_final_cols].to_csv(split_out_dir / "leakytest.csv", index=False) | |
| print(f"Saved all splits to {split_out_dir}") | |
| # make baby versions too | |
| train_exclusive[split_final_cols].sample(400, random_state=42).to_csv(split_out_dir / "babytrain.csv", index=False) | |
| val_exclusive[split_final_cols].sample(50, random_state=42).to_csv(split_out_dir / "babyval.csv", index=False) | |
| test_exclusive[split_final_cols].sample(50, random_state=42).to_csv(split_out_dir / "babytest.csv", index=False) | |
| leaky_test[split_final_cols].sample(50, random_state=42).to_csv(split_out_dir / "babyleakytest.csv", index=False) | |