PLUG-challenge / scripts /build_challenge_data.py
alanclayrichard's picture
add stacks pub link
5cc7fac
Raw History Blame Contribute Delete
19.3 kB
#!/usr/bin/env python
"""Build the challenge's public ids and private labels from plug and its Lo-Hi split.
Run it against a checkout of plug with its benchmark data downloaded, and the
Lo-Hi split directory that plug wrote (the one holding ``splits/``, ``nodes/``
and ``provenance.json``)::
python scripts/build_challenge_data.py --plug /path/to/plug \\
--split /path/to/lohi # all eleven
python scripts/build_challenge_data.py --plug /path/to/plug \\
--split /path/to/lohi megascale atlas # or a few
For each benchmark it writes, under ``--out`` (default ``.challenge-data/``)::
public/<benchmark>/manifest.parquet every row: datapoint_id, row_index,
protein id(s), [position], split
public/<benchmark>/lohi_<split>.parquet the same rows, one file per Lo-Hi split
(lohi_train, lohi_val, lohi_test, excluded)
public/<benchmark>/proteins.csv protein -> Lo-Hi split
private/ground_truth/<benchmark>.parquet every row: the manifest plus its labels
plus ``public/README.md``, the dataset card, whose ``configs`` make the Hub's viewer
show one subset per benchmark with the Lo-Hi splits as its splits.
and refreshes the committed ``data/<benchmark>/proteins.csv`` and
``data/lohi_provenance.json``.
Only identifiers and splits are public. No label, sequence, mutation or ligand
leaves ``private/``, because not every benchmark's licence allows redistributing
them (LP-PDBBind's does not); participants read the rows from the original
sources through plug's own loaders. ``private/`` goes to the private repo only
and must never be committed — ``.gitignore`` covers the output directory.
The ``datapoint_id``
--------------------
A row's position in ``plug.benchmarks.<Benchmark>.rows()`` — the same order the
Lo-Hi split file is written in, which plug pins with digests — so ``"1234"``;
for a per-residue benchmark, the row and the 1-based residue, ``"1234:56"``. A
participant reproduces it from the loader alone, with no key to re-derive.
Every row is kept, including the ones the k-cut dropped (``split == ""``): the
supervised track is scored on the ``test`` rows, the unsupervised track on all
of them.
"""
import argparse
import csv
import gzip
import hashlib
import json
import sys
from pathlib import Path
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from plug_challenge.benchmarks import ( # noqa: E402
BENCHMARK_NAMES,
GO_ASPECTS,
ID_COLUMN,
SHS27K_MODES,
SPLIT_COLUMN,
get,
)
# The per-split files of a manifest, and the value of its `split` column each holds. The
# names carry "lohi_" so the Hub's viewer cannot be read as its default "train" split.
SPLIT_FILES = {"lohi_train": "train", "lohi_val": "val", "lohi_test": "test", "excluded": ""}
PROTEIN_COLUMNS = ["node", "protein", "split"]
SPACE_URL = "https://huggingface.co/spaces/PrismScienceOrg/PLUG-challenge"
def check_public(frame: pd.DataFrame, name: str, what: str) -> None:
"""Refuse to publish anything but identifiers and splits.
The public repo is redistributed, and not every benchmark's licence allows its labels
to be (LP-PDBBind's does not). So every public file is checked against an allow-list
of columns, rather than against a list of known labels that a new one could miss.
"""
spec = get(name)
allowed = (
PROTEIN_COLUMNS
if what == "proteins"
else [ID_COLUMN, "row_index", *spec.manifest_columns, SPLIT_COLUMN]
)
extra = sorted(set(frame.columns) - set(allowed))
leaked = sorted(set(frame.columns) & set(spec.truth_columns))
if extra or leaked:
raise ValueError(
f"{name} {what}: refusing to publish columns {extra or leaked}; "
f"only {allowed} may be public"
)
def node_of(sequence: str) -> str:
"""Plug's id for a protein: a hash of its sequence (``plug.graph.node_of``)."""
return "p" + hashlib.sha1(sequence.strip().upper().encode()).hexdigest()[:16]
def read_split(split_dir: Path, name: str) -> list[str]:
"""The Lo-Hi split of every row of one benchmark, in ``rows()`` order."""
with gzip.open(split_dir / "splits" / f"{name}.csv.gz", "rt", newline="") as handle:
return [line["split"] for line in csv.DictReader(handle)]
def per_row(name: str, rows: list[dict]) -> list[dict]:
"""One record per scored datapoint: identifiers first, then labels.
Parameters
----------
name : str
Benchmark name.
rows : list[dict]
The benchmark's ``rows()``, in order.
Returns
-------
list[dict]
Each record carries ``row_index`` and, for a per-residue benchmark,
``position``; the label columns are the spec's ``truth_columns``.
"""
out: list[dict] = []
for i, row in enumerate(rows):
base = {"row_index": i}
if name in ("megascale", "fireprot"):
out.append({**base, "pdb": row["pdb"], "ddg": row["ddg"]})
elif name == "dms":
out.append(
{**base, "dms_id": row["dms_id"], "uniprot": row["uniprot"], "score": row["score"]}
)
elif name == "ppi":
out.append({**base, "id_a": row["id_a"], "id_b": row["id_b"], "label": row["label"]})
elif name == "shs27k":
modes = set(row["types"])
out.append(
{
**base,
"id_a": row["id_a"],
"id_b": row["id_b"],
**{mode: int(mode in modes) for mode in SHS27K_MODES},
}
)
elif name == "lp-pdbbind":
out.append({**base, "pdb": row["pdb"], "value": row["value"]})
elif name == "bindingdb":
# Only exact measurements are scored. A ">" or "<" row is a bound, not a value,
# so it cannot be ranked against the others; figure 4 drops it the same way.
# The row keeps its row_index, so the datapoint ids are unchanged.
if str(row.get("relation") or "").strip() != "=":
continue
out.append(
{
**base,
"target_key": node_of(row["sequence"]),
"uniprot": row["uniprot"],
"value": row["value"],
}
)
elif name in ("allobench", "passerrank"):
sites = set(row["allosteric_site"])
ids = (
{"target_id": row["target_id"], "uniprot": row["uniprot"]}
if name == "allobench"
else {"uniprot": row["uniprot"]}
)
for position in range(1, len(row["sequence"]) + 1):
out.append(
{
**base,
"position": position,
**ids,
"is_allosteric_site": int(position in sites),
}
)
elif name == "atlas":
for position, value in enumerate(row["rmsf"], start=1):
out.append(
{**base, "position": position, "pdb_chain": row["pdb_chain"], "rmsf": value}
)
else:
raise ValueError(f"no row reader for {name!r}")
return out
def build(name: str, plug: Path, split_dir: Path, out: Path, repo_root: Path) -> dict[str, int]:
"""Build every artefact for one benchmark.
Parameters
----------
name : str
Benchmark name.
plug : Path
The plug checkout whose loaders define the rows.
split_dir : Path
The Lo-Hi split directory (``splits/`` and ``nodes/`` beneath it).
out : Path
Output directory; ``public/`` and ``private/`` are created beneath it.
repo_root : Path
This repository's root, for the committed protein list.
Returns
-------
dict[str, int]
Row counts, for the summary table.
"""
sys.path.insert(0, str(plug))
from plug.benchmarks import REGISTRY # imported here so --help works without plug
spec = get(name)
rows = list(REGISTRY[name].rows())
splits = read_split(split_dir, name)
if len(rows) != len(splits):
raise ValueError(
f"the split has {len(splits):,} rows and plug's loader yields {len(rows):,}; "
"they are matched by position, so the split must be rebuilt against this data"
)
print(f"[{name}] {len(rows):,} rows from plug")
public = out / "public" / name
private = out / "private" / "ground_truth"
public.mkdir(parents=True, exist_ok=True)
private.mkdir(parents=True, exist_ok=True)
if spec.long_form:
# cafa5: one manifest row per protein, and a long-form ground truth with
# one row per (protein, branch, term) annotation.
manifest = pd.DataFrame(
{
"row_index": range(len(rows)),
"protein_id": [row["protein_id"] for row in rows],
}
)
truth = pd.DataFrame(
[
{"row_index": i, "protein_id": row["protein_id"], "aspect": aspect, "term": term}
for i, row in enumerate(rows)
for aspect in GO_ASPECTS
for term in sorted(set(row[f"go_{aspect}"]))
]
)
else:
truth = pd.DataFrame(per_row(name, rows))
manifest = truth.drop(columns=list(spec.truth_columns))
for frame in (manifest, truth):
frame.insert(0, ID_COLUMN, _datapoint_ids(frame, spec.per_residue))
frame[SPLIT_COLUMN] = [splits[i] for i in frame["row_index"]]
expected = [ID_COLUMN, "row_index", *spec.manifest_columns, SPLIT_COLUMN]
if sorted(manifest.columns) != sorted(expected):
raise ValueError(f"{name}: manifest has {list(manifest.columns)}, expected {expected}")
manifest = manifest[expected]
proteins = protein_splits(name, rows, split_dir)
check_public(manifest, name, "manifest")
check_public(proteins, name, "proteins")
manifest.to_parquet(public / "manifest.parquet", index=False)
for filename, value in SPLIT_FILES.items():
part = manifest[manifest[SPLIT_COLUMN] == value]
part.to_parquet(public / f"{filename}.parquet", index=False)
truth.to_parquet(private / f"{name}.parquet", index=False)
proteins.to_csv(public / "proteins.csv", index=False)
committed = repo_root / "data" / name
committed.mkdir(parents=True, exist_ok=True)
proteins.to_csv(committed / "proteins.csv", index=False)
test = manifest[manifest[SPLIT_COLUMN] == "test"]
counts = {
"rows": len(rows),
"datapoints": len(manifest),
"test_datapoints": len(test),
"test_rows": int(test["row_index"].nunique()),
"proteins": len(proteins),
}
print(f"[{name}] {counts}")
return counts
def _datapoint_ids(frame: pd.DataFrame, per_residue: bool) -> pd.Series:
"""``"<row_index>"``, or ``"<row_index>:<position>"`` for a per-residue benchmark."""
rows = frame["row_index"].astype(str)
if per_residue:
return (rows + ":" + frame["position"].astype(str)).astype("string")
return rows.astype("string")
def protein_splits(name: str, rows: list[dict], split_dir: Path) -> pd.DataFrame:
"""Every protein one benchmark measures, with the Lo-Hi split it was given.
Read from the split's own node file rather than re-derived from the rows, so a
protein the k-cut assigned stays listed even when none of its rows survived
(an interaction whose partner landed elsewhere), and one it gave up reads
``excluded``.
Parameters
----------
name : str
Benchmark name.
rows : list[dict]
The benchmark's rows, to name bindingdb's proteins, whose plug key is
the sequence itself.
split_dir : Path
The Lo-Hi split directory.
Returns
-------
pd.DataFrame
``node`` (plug's sequence hash, the same across benchmarks), ``protein``
(the benchmark's own id(s) for it, space-separated), ``split``.
"""
with gzip.open(split_dir / "nodes" / "proteins.csv.gz", "rt", newline="") as handle:
nodes = [line for line in csv.DictReader(handle) if line["dataset"] == name]
if name == "bindingdb":
accessions: dict[str, set[str]] = {}
for row in rows:
if row["uniprot"]:
accessions.setdefault(node_of(row["sequence"]), set()).add(row["uniprot"])
named = {node: " ".join(sorted(ids)) for node, ids in accessions.items()}
else:
prefix = f"{name}::"
named = {
line["node"]: " ".join(
entity[len(prefix):]
for entity in line["entities"].split()
if entity.startswith(prefix)
)
for line in nodes
}
return pd.DataFrame(
{
"node": [line["node"] for line in nodes],
"protein": [named.get(line["node"], "") for line in nodes],
"split": [line["split"] for line in nodes],
}
).sort_values("node", ignore_index=True)
def write_card(public: Path) -> None:
"""Write the public repo's dataset card from the manifests already in ``public``.
Without a card the Hub's viewer concatenates every manifest into one table and calls
it "train". The ``configs`` here give it one subset per benchmark, whose splits are
the per-split files, and one more subset for the protein lists.
"""
names = [name for name in BENCHMARK_NAMES if (public / name / "manifest.parquet").exists()]
yaml = [
"---",
"pretty_name: PLUG challenge — Lo-Hi ids and splits",
"license: other",
"tags: [biology, proteins, benchmark, data-leakage]",
"configs:",
]
for i, name in enumerate(names):
yaml.append(f"- config_name: {name}")
if i == 0:
yaml.append(" default: true")
yaml.append(" data_files:")
for filename in SPLIT_FILES:
yaml += [f" - split: {filename}", f" path: {name}/{filename}.parquet"]
yaml += ["- config_name: proteins", " data_files:"]
for name in names:
yaml += [f" - split: {name.replace('-', '_')}", f" path: {name}/proteins.csv"]
yaml.append("---")
table = [
"| Benchmark | Identifier columns | lohi_train | lohi_val | lohi_test | excluded |",
"|---|---|---:|---:|---:|---:|",
]
for name in names:
counts = pd.read_parquet(public / name / "manifest.parquet", columns=[SPLIT_COLUMN])[
SPLIT_COLUMN
].value_counts()
ids = ", ".join(f"`{c}`" for c in get(name).manifest_columns)
table.append(
f"| `{name}` | {ids} | "
+ " | ".join(f"{int(counts.get(v, 0)):,}" for v in SPLIT_FILES.values())
+ " |"
)
body = f"""
# PLUG challenge — Lo-Hi ids and splits
The public data of the [PLUG challenge]({SPACE_URL}): for each of {len(names)} protein
benchmarks, every datapoint's identifier and the Lo-Hi split it was given.
**There are no labels here** — see [Labels](#labels).
## Layout
One subset per benchmark. Its splits are the Lo-Hi splits:
- `lohi_train`, `lohi_val`: train and tune on these (supervised track).
- `lohi_test`: the held-out rows the supervised track is scored on.
- `excluded`: rows the Lo-Hi k-cut dropped to keep similar proteins out of different
splits. The unsupervised track scores every row, these included.
{chr(10).join(table)}
Each file has `datapoint_id`, `row_index`, the benchmark's own identifier columns, and
`split`. `<benchmark>/manifest.parquet` holds all of a benchmark's rows in one file, with
`split` empty for an excluded row. The `proteins` subset lists each benchmark's proteins
(`node`, plug's sequence hash, shared across benchmarks) with the Lo-Hi split each was
assigned; `excluded` there marks a protein the k-cut gave up.
## The `datapoint_id`
A row's position in the benchmark as plug's own loader yields it
(`plug.benchmarks.<Benchmark>.rows()`), so `"1234"` is the 1235th row. The per-residue
benchmarks (`allobench`, `passerrank`, `atlas`) add the 1-based residue: `"1234:56"`.
Submissions are keyed on it.
## Labels
This repository holds **only identifiers and split assignments**. Not every benchmark's
licence allows its measurements to be redistributed (LP-PDBBind's does not), so no label,
sequence, mutation or ligand is published here. Load the rows, labels included, from the
original sources with plug's loaders: `row_index` is the row's position there. Each
benchmark's data stays under its original licence.
## Submitting
See the [challenge Space]({SPACE_URL}) for the submission format, the two tracks and how
submissions are scored.
"""
(public / "README.md").write_text("\n".join(yaml) + "\n" + body)
def main(argv: list[str] | None = None) -> int:
"""Build the challenge data for the requested benchmarks.
Parameters
----------
argv : list[str] | None
Argument list; ``None`` reads ``sys.argv``.
Returns
-------
int
Process exit status.
"""
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument(
"benchmarks",
nargs="*",
default=list(BENCHMARK_NAMES),
help=f"Benchmarks to build (default: all of {', '.join(BENCHMARK_NAMES)})",
)
parser.add_argument("--plug", type=Path, required=True, help="plug checkout")
parser.add_argument(
"--split",
type=Path,
required=True,
help="Lo-Hi split directory (holds splits/, nodes/ and provenance.json)",
)
parser.add_argument(
"--out",
type=Path,
default=Path(".challenge-data"),
help="Output directory (default: .challenge-data, which is gitignored)",
)
args = parser.parse_args(argv)
repo_root = Path(__file__).resolve().parent.parent
summary = {}
for name in args.benchmarks:
if name not in BENCHMARK_NAMES:
print(f"skipping unknown benchmark {name!r}")
continue
try:
summary[name] = build(name, args.plug, args.split, args.out, repo_root)
except Exception as exc: # keep going: one benchmark's missing data is not fatal
print(f"[{name}] FAILED: {type(exc).__name__}: {exc}")
provenance = json.loads((args.split / "provenance.json").read_text())
record = {
"split": f"{args.split.parent.name}/{args.split.name}",
"files": provenance.get("files", {}),
}
(repo_root / "data" / "lohi_provenance.json").write_text(json.dumps(record, indent=2) + "\n")
write_card(args.out / "public")
print("\n== summary ==")
for name, counts in summary.items():
print(f"{name:12s} {counts}")
print(
f"\nPublic ids and splits: {args.out / 'public'}\n"
f"Private labels: {args.out / 'private'}\n"
"Next: upload each to its dataset repo — see docs/SETUP.md."
)
return 0 if len(summary) == len(args.benchmarks) else 1
if __name__ == "__main__":
sys.exit(main())