Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Download scripts/build_challenge_data.py from PrismScienceOrg/PLUG-challenge: direct link, hf CLI and curl.
- Browser
- Download file 19.3 kB
-
https://huggingface.co/spaces/PrismScienceOrg/PLUG-challenge/resolve/main/scripts/build_challenge_data.py
- Command line
-
hf download hf://spaces/PrismScienceOrg/PLUG-challenge/scripts/build_challenge_data.py
-
curl -L -o build_challenge_data.py https://huggingface.co/spaces/PrismScienceOrg/PLUG-challenge/resolve/main/scripts/build_challenge_data.py
19.3 kB
| #!/usr/bin/env python | |
| """Build the challenge's public ids and private labels from plug and its Lo-Hi split. | |
| Run it against a checkout of plug with its benchmark data downloaded, and the | |
| Lo-Hi split directory that plug wrote (the one holding ``splits/``, ``nodes/`` | |
| and ``provenance.json``):: | |
| python scripts/build_challenge_data.py --plug /path/to/plug \\ | |
| --split /path/to/lohi # all eleven | |
| python scripts/build_challenge_data.py --plug /path/to/plug \\ | |
| --split /path/to/lohi megascale atlas # or a few | |
| For each benchmark it writes, under ``--out`` (default ``.challenge-data/``):: | |
| public/<benchmark>/manifest.parquet every row: datapoint_id, row_index, | |
| protein id(s), [position], split | |
| public/<benchmark>/lohi_<split>.parquet the same rows, one file per Lo-Hi split | |
| (lohi_train, lohi_val, lohi_test, excluded) | |
| public/<benchmark>/proteins.csv protein -> Lo-Hi split | |
| private/ground_truth/<benchmark>.parquet every row: the manifest plus its labels | |
| plus ``public/README.md``, the dataset card, whose ``configs`` make the Hub's viewer | |
| show one subset per benchmark with the Lo-Hi splits as its splits. | |
| and refreshes the committed ``data/<benchmark>/proteins.csv`` and | |
| ``data/lohi_provenance.json``. | |
| Only identifiers and splits are public. No label, sequence, mutation or ligand | |
| leaves ``private/``, because not every benchmark's licence allows redistributing | |
| them (LP-PDBBind's does not); participants read the rows from the original | |
| sources through plug's own loaders. ``private/`` goes to the private repo only | |
| and must never be committed — ``.gitignore`` covers the output directory. | |
| The ``datapoint_id`` | |
| -------------------- | |
| A row's position in ``plug.benchmarks.<Benchmark>.rows()`` — the same order the | |
| Lo-Hi split file is written in, which plug pins with digests — so ``"1234"``; | |
| for a per-residue benchmark, the row and the 1-based residue, ``"1234:56"``. A | |
| participant reproduces it from the loader alone, with no key to re-derive. | |
| Every row is kept, including the ones the k-cut dropped (``split == ""``): the | |
| supervised track is scored on the ``test`` rows, the unsupervised track on all | |
| of them. | |
| """ | |
| import argparse | |
| import csv | |
| import gzip | |
| import hashlib | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import pandas as pd | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| from plug_challenge.benchmarks import ( # noqa: E402 | |
| BENCHMARK_NAMES, | |
| GO_ASPECTS, | |
| ID_COLUMN, | |
| SHS27K_MODES, | |
| SPLIT_COLUMN, | |
| get, | |
| ) | |
| # The per-split files of a manifest, and the value of its `split` column each holds. The | |
| # names carry "lohi_" so the Hub's viewer cannot be read as its default "train" split. | |
| SPLIT_FILES = {"lohi_train": "train", "lohi_val": "val", "lohi_test": "test", "excluded": ""} | |
| PROTEIN_COLUMNS = ["node", "protein", "split"] | |
| SPACE_URL = "https://huggingface.co/spaces/PrismScienceOrg/PLUG-challenge" | |
| def check_public(frame: pd.DataFrame, name: str, what: str) -> None: | |
| """Refuse to publish anything but identifiers and splits. | |
| The public repo is redistributed, and not every benchmark's licence allows its labels | |
| to be (LP-PDBBind's does not). So every public file is checked against an allow-list | |
| of columns, rather than against a list of known labels that a new one could miss. | |
| """ | |
| spec = get(name) | |
| allowed = ( | |
| PROTEIN_COLUMNS | |
| if what == "proteins" | |
| else [ID_COLUMN, "row_index", *spec.manifest_columns, SPLIT_COLUMN] | |
| ) | |
| extra = sorted(set(frame.columns) - set(allowed)) | |
| leaked = sorted(set(frame.columns) & set(spec.truth_columns)) | |
| if extra or leaked: | |
| raise ValueError( | |
| f"{name} {what}: refusing to publish columns {extra or leaked}; " | |
| f"only {allowed} may be public" | |
| ) | |
| def node_of(sequence: str) -> str: | |
| """Plug's id for a protein: a hash of its sequence (``plug.graph.node_of``).""" | |
| return "p" + hashlib.sha1(sequence.strip().upper().encode()).hexdigest()[:16] | |
| def read_split(split_dir: Path, name: str) -> list[str]: | |
| """The Lo-Hi split of every row of one benchmark, in ``rows()`` order.""" | |
| with gzip.open(split_dir / "splits" / f"{name}.csv.gz", "rt", newline="") as handle: | |
| return [line["split"] for line in csv.DictReader(handle)] | |
| def per_row(name: str, rows: list[dict]) -> list[dict]: | |
| """One record per scored datapoint: identifiers first, then labels. | |
| Parameters | |
| ---------- | |
| name : str | |
| Benchmark name. | |
| rows : list[dict] | |
| The benchmark's ``rows()``, in order. | |
| Returns | |
| ------- | |
| list[dict] | |
| Each record carries ``row_index`` and, for a per-residue benchmark, | |
| ``position``; the label columns are the spec's ``truth_columns``. | |
| """ | |
| out: list[dict] = [] | |
| for i, row in enumerate(rows): | |
| base = {"row_index": i} | |
| if name in ("megascale", "fireprot"): | |
| out.append({**base, "pdb": row["pdb"], "ddg": row["ddg"]}) | |
| elif name == "dms": | |
| out.append( | |
| {**base, "dms_id": row["dms_id"], "uniprot": row["uniprot"], "score": row["score"]} | |
| ) | |
| elif name == "ppi": | |
| out.append({**base, "id_a": row["id_a"], "id_b": row["id_b"], "label": row["label"]}) | |
| elif name == "shs27k": | |
| modes = set(row["types"]) | |
| out.append( | |
| { | |
| **base, | |
| "id_a": row["id_a"], | |
| "id_b": row["id_b"], | |
| **{mode: int(mode in modes) for mode in SHS27K_MODES}, | |
| } | |
| ) | |
| elif name == "lp-pdbbind": | |
| out.append({**base, "pdb": row["pdb"], "value": row["value"]}) | |
| elif name == "bindingdb": | |
| # Only exact measurements are scored. A ">" or "<" row is a bound, not a value, | |
| # so it cannot be ranked against the others; figure 4 drops it the same way. | |
| # The row keeps its row_index, so the datapoint ids are unchanged. | |
| if str(row.get("relation") or "").strip() != "=": | |
| continue | |
| out.append( | |
| { | |
| **base, | |
| "target_key": node_of(row["sequence"]), | |
| "uniprot": row["uniprot"], | |
| "value": row["value"], | |
| } | |
| ) | |
| elif name in ("allobench", "passerrank"): | |
| sites = set(row["allosteric_site"]) | |
| ids = ( | |
| {"target_id": row["target_id"], "uniprot": row["uniprot"]} | |
| if name == "allobench" | |
| else {"uniprot": row["uniprot"]} | |
| ) | |
| for position in range(1, len(row["sequence"]) + 1): | |
| out.append( | |
| { | |
| **base, | |
| "position": position, | |
| **ids, | |
| "is_allosteric_site": int(position in sites), | |
| } | |
| ) | |
| elif name == "atlas": | |
| for position, value in enumerate(row["rmsf"], start=1): | |
| out.append( | |
| {**base, "position": position, "pdb_chain": row["pdb_chain"], "rmsf": value} | |
| ) | |
| else: | |
| raise ValueError(f"no row reader for {name!r}") | |
| return out | |
| def build(name: str, plug: Path, split_dir: Path, out: Path, repo_root: Path) -> dict[str, int]: | |
| """Build every artefact for one benchmark. | |
| Parameters | |
| ---------- | |
| name : str | |
| Benchmark name. | |
| plug : Path | |
| The plug checkout whose loaders define the rows. | |
| split_dir : Path | |
| The Lo-Hi split directory (``splits/`` and ``nodes/`` beneath it). | |
| out : Path | |
| Output directory; ``public/`` and ``private/`` are created beneath it. | |
| repo_root : Path | |
| This repository's root, for the committed protein list. | |
| Returns | |
| ------- | |
| dict[str, int] | |
| Row counts, for the summary table. | |
| """ | |
| sys.path.insert(0, str(plug)) | |
| from plug.benchmarks import REGISTRY # imported here so --help works without plug | |
| spec = get(name) | |
| rows = list(REGISTRY[name].rows()) | |
| splits = read_split(split_dir, name) | |
| if len(rows) != len(splits): | |
| raise ValueError( | |
| f"the split has {len(splits):,} rows and plug's loader yields {len(rows):,}; " | |
| "they are matched by position, so the split must be rebuilt against this data" | |
| ) | |
| print(f"[{name}] {len(rows):,} rows from plug") | |
| public = out / "public" / name | |
| private = out / "private" / "ground_truth" | |
| public.mkdir(parents=True, exist_ok=True) | |
| private.mkdir(parents=True, exist_ok=True) | |
| if spec.long_form: | |
| # cafa5: one manifest row per protein, and a long-form ground truth with | |
| # one row per (protein, branch, term) annotation. | |
| manifest = pd.DataFrame( | |
| { | |
| "row_index": range(len(rows)), | |
| "protein_id": [row["protein_id"] for row in rows], | |
| } | |
| ) | |
| truth = pd.DataFrame( | |
| [ | |
| {"row_index": i, "protein_id": row["protein_id"], "aspect": aspect, "term": term} | |
| for i, row in enumerate(rows) | |
| for aspect in GO_ASPECTS | |
| for term in sorted(set(row[f"go_{aspect}"])) | |
| ] | |
| ) | |
| else: | |
| truth = pd.DataFrame(per_row(name, rows)) | |
| manifest = truth.drop(columns=list(spec.truth_columns)) | |
| for frame in (manifest, truth): | |
| frame.insert(0, ID_COLUMN, _datapoint_ids(frame, spec.per_residue)) | |
| frame[SPLIT_COLUMN] = [splits[i] for i in frame["row_index"]] | |
| expected = [ID_COLUMN, "row_index", *spec.manifest_columns, SPLIT_COLUMN] | |
| if sorted(manifest.columns) != sorted(expected): | |
| raise ValueError(f"{name}: manifest has {list(manifest.columns)}, expected {expected}") | |
| manifest = manifest[expected] | |
| proteins = protein_splits(name, rows, split_dir) | |
| check_public(manifest, name, "manifest") | |
| check_public(proteins, name, "proteins") | |
| manifest.to_parquet(public / "manifest.parquet", index=False) | |
| for filename, value in SPLIT_FILES.items(): | |
| part = manifest[manifest[SPLIT_COLUMN] == value] | |
| part.to_parquet(public / f"{filename}.parquet", index=False) | |
| truth.to_parquet(private / f"{name}.parquet", index=False) | |
| proteins.to_csv(public / "proteins.csv", index=False) | |
| committed = repo_root / "data" / name | |
| committed.mkdir(parents=True, exist_ok=True) | |
| proteins.to_csv(committed / "proteins.csv", index=False) | |
| test = manifest[manifest[SPLIT_COLUMN] == "test"] | |
| counts = { | |
| "rows": len(rows), | |
| "datapoints": len(manifest), | |
| "test_datapoints": len(test), | |
| "test_rows": int(test["row_index"].nunique()), | |
| "proteins": len(proteins), | |
| } | |
| print(f"[{name}] {counts}") | |
| return counts | |
| def _datapoint_ids(frame: pd.DataFrame, per_residue: bool) -> pd.Series: | |
| """``"<row_index>"``, or ``"<row_index>:<position>"`` for a per-residue benchmark.""" | |
| rows = frame["row_index"].astype(str) | |
| if per_residue: | |
| return (rows + ":" + frame["position"].astype(str)).astype("string") | |
| return rows.astype("string") | |
| def protein_splits(name: str, rows: list[dict], split_dir: Path) -> pd.DataFrame: | |
| """Every protein one benchmark measures, with the Lo-Hi split it was given. | |
| Read from the split's own node file rather than re-derived from the rows, so a | |
| protein the k-cut assigned stays listed even when none of its rows survived | |
| (an interaction whose partner landed elsewhere), and one it gave up reads | |
| ``excluded``. | |
| Parameters | |
| ---------- | |
| name : str | |
| Benchmark name. | |
| rows : list[dict] | |
| The benchmark's rows, to name bindingdb's proteins, whose plug key is | |
| the sequence itself. | |
| split_dir : Path | |
| The Lo-Hi split directory. | |
| Returns | |
| ------- | |
| pd.DataFrame | |
| ``node`` (plug's sequence hash, the same across benchmarks), ``protein`` | |
| (the benchmark's own id(s) for it, space-separated), ``split``. | |
| """ | |
| with gzip.open(split_dir / "nodes" / "proteins.csv.gz", "rt", newline="") as handle: | |
| nodes = [line for line in csv.DictReader(handle) if line["dataset"] == name] | |
| if name == "bindingdb": | |
| accessions: dict[str, set[str]] = {} | |
| for row in rows: | |
| if row["uniprot"]: | |
| accessions.setdefault(node_of(row["sequence"]), set()).add(row["uniprot"]) | |
| named = {node: " ".join(sorted(ids)) for node, ids in accessions.items()} | |
| else: | |
| prefix = f"{name}::" | |
| named = { | |
| line["node"]: " ".join( | |
| entity[len(prefix):] | |
| for entity in line["entities"].split() | |
| if entity.startswith(prefix) | |
| ) | |
| for line in nodes | |
| } | |
| return pd.DataFrame( | |
| { | |
| "node": [line["node"] for line in nodes], | |
| "protein": [named.get(line["node"], "") for line in nodes], | |
| "split": [line["split"] for line in nodes], | |
| } | |
| ).sort_values("node", ignore_index=True) | |
| def write_card(public: Path) -> None: | |
| """Write the public repo's dataset card from the manifests already in ``public``. | |
| Without a card the Hub's viewer concatenates every manifest into one table and calls | |
| it "train". The ``configs`` here give it one subset per benchmark, whose splits are | |
| the per-split files, and one more subset for the protein lists. | |
| """ | |
| names = [name for name in BENCHMARK_NAMES if (public / name / "manifest.parquet").exists()] | |
| yaml = [ | |
| "---", | |
| "pretty_name: PLUG challenge — Lo-Hi ids and splits", | |
| "license: other", | |
| "tags: [biology, proteins, benchmark, data-leakage]", | |
| "configs:", | |
| ] | |
| for i, name in enumerate(names): | |
| yaml.append(f"- config_name: {name}") | |
| if i == 0: | |
| yaml.append(" default: true") | |
| yaml.append(" data_files:") | |
| for filename in SPLIT_FILES: | |
| yaml += [f" - split: {filename}", f" path: {name}/{filename}.parquet"] | |
| yaml += ["- config_name: proteins", " data_files:"] | |
| for name in names: | |
| yaml += [f" - split: {name.replace('-', '_')}", f" path: {name}/proteins.csv"] | |
| yaml.append("---") | |
| table = [ | |
| "| Benchmark | Identifier columns | lohi_train | lohi_val | lohi_test | excluded |", | |
| "|---|---|---:|---:|---:|---:|", | |
| ] | |
| for name in names: | |
| counts = pd.read_parquet(public / name / "manifest.parquet", columns=[SPLIT_COLUMN])[ | |
| SPLIT_COLUMN | |
| ].value_counts() | |
| ids = ", ".join(f"`{c}`" for c in get(name).manifest_columns) | |
| table.append( | |
| f"| `{name}` | {ids} | " | |
| + " | ".join(f"{int(counts.get(v, 0)):,}" for v in SPLIT_FILES.values()) | |
| + " |" | |
| ) | |
| body = f""" | |
| # PLUG challenge — Lo-Hi ids and splits | |
| The public data of the [PLUG challenge]({SPACE_URL}): for each of {len(names)} protein | |
| benchmarks, every datapoint's identifier and the Lo-Hi split it was given. | |
| **There are no labels here** — see [Labels](#labels). | |
| ## Layout | |
| One subset per benchmark. Its splits are the Lo-Hi splits: | |
| - `lohi_train`, `lohi_val`: train and tune on these (supervised track). | |
| - `lohi_test`: the held-out rows the supervised track is scored on. | |
| - `excluded`: rows the Lo-Hi k-cut dropped to keep similar proteins out of different | |
| splits. The unsupervised track scores every row, these included. | |
| {chr(10).join(table)} | |
| Each file has `datapoint_id`, `row_index`, the benchmark's own identifier columns, and | |
| `split`. `<benchmark>/manifest.parquet` holds all of a benchmark's rows in one file, with | |
| `split` empty for an excluded row. The `proteins` subset lists each benchmark's proteins | |
| (`node`, plug's sequence hash, shared across benchmarks) with the Lo-Hi split each was | |
| assigned; `excluded` there marks a protein the k-cut gave up. | |
| ## The `datapoint_id` | |
| A row's position in the benchmark as plug's own loader yields it | |
| (`plug.benchmarks.<Benchmark>.rows()`), so `"1234"` is the 1235th row. The per-residue | |
| benchmarks (`allobench`, `passerrank`, `atlas`) add the 1-based residue: `"1234:56"`. | |
| Submissions are keyed on it. | |
| ## Labels | |
| This repository holds **only identifiers and split assignments**. Not every benchmark's | |
| licence allows its measurements to be redistributed (LP-PDBBind's does not), so no label, | |
| sequence, mutation or ligand is published here. Load the rows, labels included, from the | |
| original sources with plug's loaders: `row_index` is the row's position there. Each | |
| benchmark's data stays under its original licence. | |
| ## Submitting | |
| See the [challenge Space]({SPACE_URL}) for the submission format, the two tracks and how | |
| submissions are scored. | |
| """ | |
| (public / "README.md").write_text("\n".join(yaml) + "\n" + body) | |
| def main(argv: list[str] | None = None) -> int: | |
| """Build the challenge data for the requested benchmarks. | |
| Parameters | |
| ---------- | |
| argv : list[str] | None | |
| Argument list; ``None`` reads ``sys.argv``. | |
| Returns | |
| ------- | |
| int | |
| Process exit status. | |
| """ | |
| parser = argparse.ArgumentParser( | |
| description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter | |
| ) | |
| parser.add_argument( | |
| "benchmarks", | |
| nargs="*", | |
| default=list(BENCHMARK_NAMES), | |
| help=f"Benchmarks to build (default: all of {', '.join(BENCHMARK_NAMES)})", | |
| ) | |
| parser.add_argument("--plug", type=Path, required=True, help="plug checkout") | |
| parser.add_argument( | |
| "--split", | |
| type=Path, | |
| required=True, | |
| help="Lo-Hi split directory (holds splits/, nodes/ and provenance.json)", | |
| ) | |
| parser.add_argument( | |
| "--out", | |
| type=Path, | |
| default=Path(".challenge-data"), | |
| help="Output directory (default: .challenge-data, which is gitignored)", | |
| ) | |
| args = parser.parse_args(argv) | |
| repo_root = Path(__file__).resolve().parent.parent | |
| summary = {} | |
| for name in args.benchmarks: | |
| if name not in BENCHMARK_NAMES: | |
| print(f"skipping unknown benchmark {name!r}") | |
| continue | |
| try: | |
| summary[name] = build(name, args.plug, args.split, args.out, repo_root) | |
| except Exception as exc: # keep going: one benchmark's missing data is not fatal | |
| print(f"[{name}] FAILED: {type(exc).__name__}: {exc}") | |
| provenance = json.loads((args.split / "provenance.json").read_text()) | |
| record = { | |
| "split": f"{args.split.parent.name}/{args.split.name}", | |
| "files": provenance.get("files", {}), | |
| } | |
| (repo_root / "data" / "lohi_provenance.json").write_text(json.dumps(record, indent=2) + "\n") | |
| write_card(args.out / "public") | |
| print("\n== summary ==") | |
| for name, counts in summary.items(): | |
| print(f"{name:12s} {counts}") | |
| print( | |
| f"\nPublic ids and splits: {args.out / 'public'}\n" | |
| f"Private labels: {args.out / 'private'}\n" | |
| "Next: upload each to its dataset repo — see docs/SETUP.md." | |
| ) | |
| return 0 if len(summary) == len(args.benchmarks) else 1 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |