#!/usr/bin/env python """Build the challenge's public ids and private labels from plug and its Lo-Hi split. Run it against a checkout of plug with its benchmark data downloaded, and the Lo-Hi split directory that plug wrote (the one holding ``splits/``, ``nodes/`` and ``provenance.json``):: python scripts/build_challenge_data.py --plug /path/to/plug \\ --split /path/to/lohi # all eleven python scripts/build_challenge_data.py --plug /path/to/plug \\ --split /path/to/lohi megascale atlas # or a few For each benchmark it writes, under ``--out`` (default ``.challenge-data/``):: public//manifest.parquet every row: datapoint_id, row_index, protein id(s), [position], split public//lohi_.parquet the same rows, one file per Lo-Hi split (lohi_train, lohi_val, lohi_test, excluded) public//proteins.csv protein -> Lo-Hi split private/ground_truth/.parquet every row: the manifest plus its labels plus ``public/README.md``, the dataset card, whose ``configs`` make the Hub's viewer show one subset per benchmark with the Lo-Hi splits as its splits. and refreshes the committed ``data//proteins.csv`` and ``data/lohi_provenance.json``. Only identifiers and splits are public. No label, sequence, mutation or ligand leaves ``private/``, because not every benchmark's licence allows redistributing them (LP-PDBBind's does not); participants read the rows from the original sources through plug's own loaders. ``private/`` goes to the private repo only and must never be committed — ``.gitignore`` covers the output directory. The ``datapoint_id`` -------------------- A row's position in ``plug.benchmarks..rows()`` — the same order the Lo-Hi split file is written in, which plug pins with digests — so ``"1234"``; for a per-residue benchmark, the row and the 1-based residue, ``"1234:56"``. A participant reproduces it from the loader alone, with no key to re-derive. Every row is kept, including the ones the k-cut dropped (``split == ""``): the supervised track is scored on the ``test`` rows, the unsupervised track on all of them. """ import argparse import csv import gzip import hashlib import json import sys from pathlib import Path import pandas as pd sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from plug_challenge.benchmarks import ( # noqa: E402 BENCHMARK_NAMES, GO_ASPECTS, ID_COLUMN, SHS27K_MODES, SPLIT_COLUMN, get, ) # The per-split files of a manifest, and the value of its `split` column each holds. The # names carry "lohi_" so the Hub's viewer cannot be read as its default "train" split. SPLIT_FILES = {"lohi_train": "train", "lohi_val": "val", "lohi_test": "test", "excluded": ""} PROTEIN_COLUMNS = ["node", "protein", "split"] SPACE_URL = "https://huggingface.co/spaces/PrismScienceOrg/PLUG-challenge" def check_public(frame: pd.DataFrame, name: str, what: str) -> None: """Refuse to publish anything but identifiers and splits. The public repo is redistributed, and not every benchmark's licence allows its labels to be (LP-PDBBind's does not). So every public file is checked against an allow-list of columns, rather than against a list of known labels that a new one could miss. """ spec = get(name) allowed = ( PROTEIN_COLUMNS if what == "proteins" else [ID_COLUMN, "row_index", *spec.manifest_columns, SPLIT_COLUMN] ) extra = sorted(set(frame.columns) - set(allowed)) leaked = sorted(set(frame.columns) & set(spec.truth_columns)) if extra or leaked: raise ValueError( f"{name} {what}: refusing to publish columns {extra or leaked}; " f"only {allowed} may be public" ) def node_of(sequence: str) -> str: """Plug's id for a protein: a hash of its sequence (``plug.graph.node_of``).""" return "p" + hashlib.sha1(sequence.strip().upper().encode()).hexdigest()[:16] def read_split(split_dir: Path, name: str) -> list[str]: """The Lo-Hi split of every row of one benchmark, in ``rows()`` order.""" with gzip.open(split_dir / "splits" / f"{name}.csv.gz", "rt", newline="") as handle: return [line["split"] for line in csv.DictReader(handle)] def per_row(name: str, rows: list[dict]) -> list[dict]: """One record per scored datapoint: identifiers first, then labels. Parameters ---------- name : str Benchmark name. rows : list[dict] The benchmark's ``rows()``, in order. Returns ------- list[dict] Each record carries ``row_index`` and, for a per-residue benchmark, ``position``; the label columns are the spec's ``truth_columns``. """ out: list[dict] = [] for i, row in enumerate(rows): base = {"row_index": i} if name in ("megascale", "fireprot"): out.append({**base, "pdb": row["pdb"], "ddg": row["ddg"]}) elif name == "dms": out.append( {**base, "dms_id": row["dms_id"], "uniprot": row["uniprot"], "score": row["score"]} ) elif name == "ppi": out.append({**base, "id_a": row["id_a"], "id_b": row["id_b"], "label": row["label"]}) elif name == "shs27k": modes = set(row["types"]) out.append( { **base, "id_a": row["id_a"], "id_b": row["id_b"], **{mode: int(mode in modes) for mode in SHS27K_MODES}, } ) elif name == "lp-pdbbind": out.append({**base, "pdb": row["pdb"], "value": row["value"]}) elif name == "bindingdb": # Only exact measurements are scored. A ">" or "<" row is a bound, not a value, # so it cannot be ranked against the others; figure 4 drops it the same way. # The row keeps its row_index, so the datapoint ids are unchanged. if str(row.get("relation") or "").strip() != "=": continue out.append( { **base, "target_key": node_of(row["sequence"]), "uniprot": row["uniprot"], "value": row["value"], } ) elif name in ("allobench", "passerrank"): sites = set(row["allosteric_site"]) ids = ( {"target_id": row["target_id"], "uniprot": row["uniprot"]} if name == "allobench" else {"uniprot": row["uniprot"]} ) for position in range(1, len(row["sequence"]) + 1): out.append( { **base, "position": position, **ids, "is_allosteric_site": int(position in sites), } ) elif name == "atlas": for position, value in enumerate(row["rmsf"], start=1): out.append( {**base, "position": position, "pdb_chain": row["pdb_chain"], "rmsf": value} ) else: raise ValueError(f"no row reader for {name!r}") return out def build(name: str, plug: Path, split_dir: Path, out: Path, repo_root: Path) -> dict[str, int]: """Build every artefact for one benchmark. Parameters ---------- name : str Benchmark name. plug : Path The plug checkout whose loaders define the rows. split_dir : Path The Lo-Hi split directory (``splits/`` and ``nodes/`` beneath it). out : Path Output directory; ``public/`` and ``private/`` are created beneath it. repo_root : Path This repository's root, for the committed protein list. Returns ------- dict[str, int] Row counts, for the summary table. """ sys.path.insert(0, str(plug)) from plug.benchmarks import REGISTRY # imported here so --help works without plug spec = get(name) rows = list(REGISTRY[name].rows()) splits = read_split(split_dir, name) if len(rows) != len(splits): raise ValueError( f"the split has {len(splits):,} rows and plug's loader yields {len(rows):,}; " "they are matched by position, so the split must be rebuilt against this data" ) print(f"[{name}] {len(rows):,} rows from plug") public = out / "public" / name private = out / "private" / "ground_truth" public.mkdir(parents=True, exist_ok=True) private.mkdir(parents=True, exist_ok=True) if spec.long_form: # cafa5: one manifest row per protein, and a long-form ground truth with # one row per (protein, branch, term) annotation. manifest = pd.DataFrame( { "row_index": range(len(rows)), "protein_id": [row["protein_id"] for row in rows], } ) truth = pd.DataFrame( [ {"row_index": i, "protein_id": row["protein_id"], "aspect": aspect, "term": term} for i, row in enumerate(rows) for aspect in GO_ASPECTS for term in sorted(set(row[f"go_{aspect}"])) ] ) else: truth = pd.DataFrame(per_row(name, rows)) manifest = truth.drop(columns=list(spec.truth_columns)) for frame in (manifest, truth): frame.insert(0, ID_COLUMN, _datapoint_ids(frame, spec.per_residue)) frame[SPLIT_COLUMN] = [splits[i] for i in frame["row_index"]] expected = [ID_COLUMN, "row_index", *spec.manifest_columns, SPLIT_COLUMN] if sorted(manifest.columns) != sorted(expected): raise ValueError(f"{name}: manifest has {list(manifest.columns)}, expected {expected}") manifest = manifest[expected] proteins = protein_splits(name, rows, split_dir) check_public(manifest, name, "manifest") check_public(proteins, name, "proteins") manifest.to_parquet(public / "manifest.parquet", index=False) for filename, value in SPLIT_FILES.items(): part = manifest[manifest[SPLIT_COLUMN] == value] part.to_parquet(public / f"{filename}.parquet", index=False) truth.to_parquet(private / f"{name}.parquet", index=False) proteins.to_csv(public / "proteins.csv", index=False) committed = repo_root / "data" / name committed.mkdir(parents=True, exist_ok=True) proteins.to_csv(committed / "proteins.csv", index=False) test = manifest[manifest[SPLIT_COLUMN] == "test"] counts = { "rows": len(rows), "datapoints": len(manifest), "test_datapoints": len(test), "test_rows": int(test["row_index"].nunique()), "proteins": len(proteins), } print(f"[{name}] {counts}") return counts def _datapoint_ids(frame: pd.DataFrame, per_residue: bool) -> pd.Series: """``""``, or ``":"`` for a per-residue benchmark.""" rows = frame["row_index"].astype(str) if per_residue: return (rows + ":" + frame["position"].astype(str)).astype("string") return rows.astype("string") def protein_splits(name: str, rows: list[dict], split_dir: Path) -> pd.DataFrame: """Every protein one benchmark measures, with the Lo-Hi split it was given. Read from the split's own node file rather than re-derived from the rows, so a protein the k-cut assigned stays listed even when none of its rows survived (an interaction whose partner landed elsewhere), and one it gave up reads ``excluded``. Parameters ---------- name : str Benchmark name. rows : list[dict] The benchmark's rows, to name bindingdb's proteins, whose plug key is the sequence itself. split_dir : Path The Lo-Hi split directory. Returns ------- pd.DataFrame ``node`` (plug's sequence hash, the same across benchmarks), ``protein`` (the benchmark's own id(s) for it, space-separated), ``split``. """ with gzip.open(split_dir / "nodes" / "proteins.csv.gz", "rt", newline="") as handle: nodes = [line for line in csv.DictReader(handle) if line["dataset"] == name] if name == "bindingdb": accessions: dict[str, set[str]] = {} for row in rows: if row["uniprot"]: accessions.setdefault(node_of(row["sequence"]), set()).add(row["uniprot"]) named = {node: " ".join(sorted(ids)) for node, ids in accessions.items()} else: prefix = f"{name}::" named = { line["node"]: " ".join( entity[len(prefix):] for entity in line["entities"].split() if entity.startswith(prefix) ) for line in nodes } return pd.DataFrame( { "node": [line["node"] for line in nodes], "protein": [named.get(line["node"], "") for line in nodes], "split": [line["split"] for line in nodes], } ).sort_values("node", ignore_index=True) def write_card(public: Path) -> None: """Write the public repo's dataset card from the manifests already in ``public``. Without a card the Hub's viewer concatenates every manifest into one table and calls it "train". The ``configs`` here give it one subset per benchmark, whose splits are the per-split files, and one more subset for the protein lists. """ names = [name for name in BENCHMARK_NAMES if (public / name / "manifest.parquet").exists()] yaml = [ "---", "pretty_name: PLUG challenge — Lo-Hi ids and splits", "license: other", "tags: [biology, proteins, benchmark, data-leakage]", "configs:", ] for i, name in enumerate(names): yaml.append(f"- config_name: {name}") if i == 0: yaml.append(" default: true") yaml.append(" data_files:") for filename in SPLIT_FILES: yaml += [f" - split: {filename}", f" path: {name}/{filename}.parquet"] yaml += ["- config_name: proteins", " data_files:"] for name in names: yaml += [f" - split: {name.replace('-', '_')}", f" path: {name}/proteins.csv"] yaml.append("---") table = [ "| Benchmark | Identifier columns | lohi_train | lohi_val | lohi_test | excluded |", "|---|---|---:|---:|---:|---:|", ] for name in names: counts = pd.read_parquet(public / name / "manifest.parquet", columns=[SPLIT_COLUMN])[ SPLIT_COLUMN ].value_counts() ids = ", ".join(f"`{c}`" for c in get(name).manifest_columns) table.append( f"| `{name}` | {ids} | " + " | ".join(f"{int(counts.get(v, 0)):,}" for v in SPLIT_FILES.values()) + " |" ) body = f""" # PLUG challenge — Lo-Hi ids and splits The public data of the [PLUG challenge]({SPACE_URL}): for each of {len(names)} protein benchmarks, every datapoint's identifier and the Lo-Hi split it was given. **There are no labels here** — see [Labels](#labels). ## Layout One subset per benchmark. Its splits are the Lo-Hi splits: - `lohi_train`, `lohi_val`: train and tune on these (supervised track). - `lohi_test`: the held-out rows the supervised track is scored on. - `excluded`: rows the Lo-Hi k-cut dropped to keep similar proteins out of different splits. The unsupervised track scores every row, these included. {chr(10).join(table)} Each file has `datapoint_id`, `row_index`, the benchmark's own identifier columns, and `split`. `/manifest.parquet` holds all of a benchmark's rows in one file, with `split` empty for an excluded row. The `proteins` subset lists each benchmark's proteins (`node`, plug's sequence hash, shared across benchmarks) with the Lo-Hi split each was assigned; `excluded` there marks a protein the k-cut gave up. ## The `datapoint_id` A row's position in the benchmark as plug's own loader yields it (`plug.benchmarks..rows()`), so `"1234"` is the 1235th row. The per-residue benchmarks (`allobench`, `passerrank`, `atlas`) add the 1-based residue: `"1234:56"`. Submissions are keyed on it. ## Labels This repository holds **only identifiers and split assignments**. Not every benchmark's licence allows its measurements to be redistributed (LP-PDBBind's does not), so no label, sequence, mutation or ligand is published here. Load the rows, labels included, from the original sources with plug's loaders: `row_index` is the row's position there. Each benchmark's data stays under its original licence. ## Submitting See the [challenge Space]({SPACE_URL}) for the submission format, the two tracks and how submissions are scored. """ (public / "README.md").write_text("\n".join(yaml) + "\n" + body) def main(argv: list[str] | None = None) -> int: """Build the challenge data for the requested benchmarks. Parameters ---------- argv : list[str] | None Argument list; ``None`` reads ``sys.argv``. Returns ------- int Process exit status. """ parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument( "benchmarks", nargs="*", default=list(BENCHMARK_NAMES), help=f"Benchmarks to build (default: all of {', '.join(BENCHMARK_NAMES)})", ) parser.add_argument("--plug", type=Path, required=True, help="plug checkout") parser.add_argument( "--split", type=Path, required=True, help="Lo-Hi split directory (holds splits/, nodes/ and provenance.json)", ) parser.add_argument( "--out", type=Path, default=Path(".challenge-data"), help="Output directory (default: .challenge-data, which is gitignored)", ) args = parser.parse_args(argv) repo_root = Path(__file__).resolve().parent.parent summary = {} for name in args.benchmarks: if name not in BENCHMARK_NAMES: print(f"skipping unknown benchmark {name!r}") continue try: summary[name] = build(name, args.plug, args.split, args.out, repo_root) except Exception as exc: # keep going: one benchmark's missing data is not fatal print(f"[{name}] FAILED: {type(exc).__name__}: {exc}") provenance = json.loads((args.split / "provenance.json").read_text()) record = { "split": f"{args.split.parent.name}/{args.split.name}", "files": provenance.get("files", {}), } (repo_root / "data" / "lohi_provenance.json").write_text(json.dumps(record, indent=2) + "\n") write_card(args.out / "public") print("\n== summary ==") for name, counts in summary.items(): print(f"{name:12s} {counts}") print( f"\nPublic ids and splits: {args.out / 'public'}\n" f"Private labels: {args.out / 'private'}\n" "Next: upload each to its dataset repo — see docs/SETUP.md." ) return 0 if len(summary) == len(args.benchmarks) else 1 if __name__ == "__main__": sys.exit(main())