File size: 19,294 Bytes
b78ff28
49442cb
b78ff28
49442cb
 
 
b78ff28
49442cb
 
 
 
b78ff28
 
 
49442cb
 
5cc7fac
 
49442cb
 
 
5cc7fac
 
 
49442cb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b78ff28
 
 
49442cb
 
b78ff28
49442cb
b78ff28
 
 
 
 
 
 
 
 
 
 
 
49442cb
b78ff28
 
 
5cc7fac
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b78ff28
49442cb
 
 
b78ff28
 
49442cb
 
 
 
b78ff28
 
49442cb
 
b78ff28
 
 
 
 
49442cb
 
b78ff28
 
 
49442cb
 
 
b78ff28
 
49442cb
 
 
 
 
 
 
 
 
 
 
 
 
 
b78ff28
49442cb
 
 
 
b78ff28
49442cb
 
 
 
580f4e5
 
 
 
 
49442cb
b78ff28
49442cb
 
 
 
b78ff28
 
49442cb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b78ff28
 
 
 
 
 
49442cb
 
 
 
b78ff28
 
 
49442cb
b78ff28
 
 
 
49442cb
b78ff28
 
49442cb
 
b78ff28
 
49442cb
 
 
 
 
 
 
 
b78ff28
 
 
 
 
 
 
49442cb
 
b78ff28
 
49442cb
 
b78ff28
 
49442cb
 
 
 
 
 
 
b78ff28
 
49442cb
 
 
 
 
 
 
 
 
 
 
5cc7fac
 
 
49442cb
 
5cc7fac
 
 
49442cb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b78ff28
 
 
49442cb
 
 
 
 
 
b78ff28
 
49442cb
 
 
 
 
 
 
b78ff28
 
 
 
 
49442cb
 
 
 
 
 
 
 
 
 
 
b78ff28
 
49442cb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b78ff28
 
5cc7fac
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b78ff28
 
 
 
 
 
 
 
 
 
 
 
 
 
49442cb
 
 
b78ff28
 
 
 
 
 
49442cb
b78ff28
49442cb
b78ff28
49442cb
 
b78ff28
 
49442cb
b78ff28
49442cb
 
b78ff28
 
 
 
 
 
 
 
 
 
49442cb
b78ff28
 
 
49442cb
 
 
 
 
 
5cc7fac
49442cb
b78ff28
 
 
 
49442cb
 
b78ff28
 
49442cb
b78ff28
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
#!/usr/bin/env python
"""Build the challenge's public ids and private labels from plug and its Lo-Hi split.

Run it against a checkout of plug with its benchmark data downloaded, and the
Lo-Hi split directory that plug wrote (the one holding ``splits/``, ``nodes/``
and ``provenance.json``)::

    python scripts/build_challenge_data.py --plug /path/to/plug \\
        --split /path/to/lohi                       # all eleven
    python scripts/build_challenge_data.py --plug /path/to/plug \\
        --split /path/to/lohi megascale atlas       # or a few

For each benchmark it writes, under ``--out`` (default ``.challenge-data/``)::

    public/<benchmark>/manifest.parquet      every row: datapoint_id, row_index,
                                             protein id(s), [position], split
    public/<benchmark>/lohi_<split>.parquet  the same rows, one file per Lo-Hi split
                                             (lohi_train, lohi_val, lohi_test, excluded)
    public/<benchmark>/proteins.csv          protein -> Lo-Hi split
    private/ground_truth/<benchmark>.parquet every row: the manifest plus its labels

plus ``public/README.md``, the dataset card, whose ``configs`` make the Hub's viewer
show one subset per benchmark with the Lo-Hi splits as its splits.

and refreshes the committed ``data/<benchmark>/proteins.csv`` and
``data/lohi_provenance.json``.

Only identifiers and splits are public. No label, sequence, mutation or ligand
leaves ``private/``, because not every benchmark's licence allows redistributing
them (LP-PDBBind's does not); participants read the rows from the original
sources through plug's own loaders. ``private/`` goes to the private repo only
and must never be committed — ``.gitignore`` covers the output directory.

The ``datapoint_id``
--------------------
A row's position in ``plug.benchmarks.<Benchmark>.rows()`` — the same order the
Lo-Hi split file is written in, which plug pins with digests — so ``"1234"``;
for a per-residue benchmark, the row and the 1-based residue, ``"1234:56"``. A
participant reproduces it from the loader alone, with no key to re-derive.

Every row is kept, including the ones the k-cut dropped (``split == ""``): the
supervised track is scored on the ``test`` rows, the unsupervised track on all
of them.
"""

import argparse
import csv
import gzip
import hashlib
import json
import sys
from pathlib import Path

import pandas as pd

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from plug_challenge.benchmarks import (  # noqa: E402
    BENCHMARK_NAMES,
    GO_ASPECTS,
    ID_COLUMN,
    SHS27K_MODES,
    SPLIT_COLUMN,
    get,
)

# The per-split files of a manifest, and the value of its `split` column each holds. The
# names carry "lohi_" so the Hub's viewer cannot be read as its default "train" split.
SPLIT_FILES = {"lohi_train": "train", "lohi_val": "val", "lohi_test": "test", "excluded": ""}
PROTEIN_COLUMNS = ["node", "protein", "split"]
SPACE_URL = "https://huggingface.co/spaces/PrismScienceOrg/PLUG-challenge"


def check_public(frame: pd.DataFrame, name: str, what: str) -> None:
    """Refuse to publish anything but identifiers and splits.

    The public repo is redistributed, and not every benchmark's licence allows its labels
    to be (LP-PDBBind's does not). So every public file is checked against an allow-list
    of columns, rather than against a list of known labels that a new one could miss.
    """
    spec = get(name)
    allowed = (
        PROTEIN_COLUMNS
        if what == "proteins"
        else [ID_COLUMN, "row_index", *spec.manifest_columns, SPLIT_COLUMN]
    )
    extra = sorted(set(frame.columns) - set(allowed))
    leaked = sorted(set(frame.columns) & set(spec.truth_columns))
    if extra or leaked:
        raise ValueError(
            f"{name} {what}: refusing to publish columns {extra or leaked}; "
            f"only {allowed} may be public"
        )


def node_of(sequence: str) -> str:
    """Plug's id for a protein: a hash of its sequence (``plug.graph.node_of``)."""
    return "p" + hashlib.sha1(sequence.strip().upper().encode()).hexdigest()[:16]


def read_split(split_dir: Path, name: str) -> list[str]:
    """The Lo-Hi split of every row of one benchmark, in ``rows()`` order."""
    with gzip.open(split_dir / "splits" / f"{name}.csv.gz", "rt", newline="") as handle:
        return [line["split"] for line in csv.DictReader(handle)]


def per_row(name: str, rows: list[dict]) -> list[dict]:
    """One record per scored datapoint: identifiers first, then labels.

    Parameters
    ----------
    name : str
        Benchmark name.
    rows : list[dict]
        The benchmark's ``rows()``, in order.

    Returns
    -------
    list[dict]
        Each record carries ``row_index`` and, for a per-residue benchmark,
        ``position``; the label columns are the spec's ``truth_columns``.

    """
    out: list[dict] = []
    for i, row in enumerate(rows):
        base = {"row_index": i}
        if name in ("megascale", "fireprot"):
            out.append({**base, "pdb": row["pdb"], "ddg": row["ddg"]})
        elif name == "dms":
            out.append(
                {**base, "dms_id": row["dms_id"], "uniprot": row["uniprot"], "score": row["score"]}
            )
        elif name == "ppi":
            out.append({**base, "id_a": row["id_a"], "id_b": row["id_b"], "label": row["label"]})
        elif name == "shs27k":
            modes = set(row["types"])
            out.append(
                {
                    **base,
                    "id_a": row["id_a"],
                    "id_b": row["id_b"],
                    **{mode: int(mode in modes) for mode in SHS27K_MODES},
                }
            )
        elif name == "lp-pdbbind":
            out.append({**base, "pdb": row["pdb"], "value": row["value"]})
        elif name == "bindingdb":
            # Only exact measurements are scored. A ">" or "<" row is a bound, not a value,
            # so it cannot be ranked against the others; figure 4 drops it the same way.
            # The row keeps its row_index, so the datapoint ids are unchanged.
            if str(row.get("relation") or "").strip() != "=":
                continue
            out.append(
                {
                    **base,
                    "target_key": node_of(row["sequence"]),
                    "uniprot": row["uniprot"],
                    "value": row["value"],
                }
            )
        elif name in ("allobench", "passerrank"):
            sites = set(row["allosteric_site"])
            ids = (
                {"target_id": row["target_id"], "uniprot": row["uniprot"]}
                if name == "allobench"
                else {"uniprot": row["uniprot"]}
            )
            for position in range(1, len(row["sequence"]) + 1):
                out.append(
                    {
                        **base,
                        "position": position,
                        **ids,
                        "is_allosteric_site": int(position in sites),
                    }
                )
        elif name == "atlas":
            for position, value in enumerate(row["rmsf"], start=1):
                out.append(
                    {**base, "position": position, "pdb_chain": row["pdb_chain"], "rmsf": value}
                )
        else:
            raise ValueError(f"no row reader for {name!r}")
    return out


def build(name: str, plug: Path, split_dir: Path, out: Path, repo_root: Path) -> dict[str, int]:
    """Build every artefact for one benchmark.

    Parameters
    ----------
    name : str
        Benchmark name.
    plug : Path
        The plug checkout whose loaders define the rows.
    split_dir : Path
        The Lo-Hi split directory (``splits/`` and ``nodes/`` beneath it).
    out : Path
        Output directory; ``public/`` and ``private/`` are created beneath it.
    repo_root : Path
        This repository's root, for the committed protein list.

    Returns
    -------
    dict[str, int]
        Row counts, for the summary table.

    """
    sys.path.insert(0, str(plug))
    from plug.benchmarks import REGISTRY  # imported here so --help works without plug

    spec = get(name)
    rows = list(REGISTRY[name].rows())
    splits = read_split(split_dir, name)
    if len(rows) != len(splits):
        raise ValueError(
            f"the split has {len(splits):,} rows and plug's loader yields {len(rows):,}; "
            "they are matched by position, so the split must be rebuilt against this data"
        )
    print(f"[{name}] {len(rows):,} rows from plug")

    public = out / "public" / name
    private = out / "private" / "ground_truth"
    public.mkdir(parents=True, exist_ok=True)
    private.mkdir(parents=True, exist_ok=True)

    if spec.long_form:
        # cafa5: one manifest row per protein, and a long-form ground truth with
        # one row per (protein, branch, term) annotation.
        manifest = pd.DataFrame(
            {
                "row_index": range(len(rows)),
                "protein_id": [row["protein_id"] for row in rows],
            }
        )
        truth = pd.DataFrame(
            [
                {"row_index": i, "protein_id": row["protein_id"], "aspect": aspect, "term": term}
                for i, row in enumerate(rows)
                for aspect in GO_ASPECTS
                for term in sorted(set(row[f"go_{aspect}"]))
            ]
        )
    else:
        truth = pd.DataFrame(per_row(name, rows))
        manifest = truth.drop(columns=list(spec.truth_columns))

    for frame in (manifest, truth):
        frame.insert(0, ID_COLUMN, _datapoint_ids(frame, spec.per_residue))
        frame[SPLIT_COLUMN] = [splits[i] for i in frame["row_index"]]

    expected = [ID_COLUMN, "row_index", *spec.manifest_columns, SPLIT_COLUMN]
    if sorted(manifest.columns) != sorted(expected):
        raise ValueError(f"{name}: manifest has {list(manifest.columns)}, expected {expected}")
    manifest = manifest[expected]
    proteins = protein_splits(name, rows, split_dir)
    check_public(manifest, name, "manifest")
    check_public(proteins, name, "proteins")

    manifest.to_parquet(public / "manifest.parquet", index=False)
    for filename, value in SPLIT_FILES.items():
        part = manifest[manifest[SPLIT_COLUMN] == value]
        part.to_parquet(public / f"{filename}.parquet", index=False)
    truth.to_parquet(private / f"{name}.parquet", index=False)
    proteins.to_csv(public / "proteins.csv", index=False)
    committed = repo_root / "data" / name
    committed.mkdir(parents=True, exist_ok=True)
    proteins.to_csv(committed / "proteins.csv", index=False)

    test = manifest[manifest[SPLIT_COLUMN] == "test"]
    counts = {
        "rows": len(rows),
        "datapoints": len(manifest),
        "test_datapoints": len(test),
        "test_rows": int(test["row_index"].nunique()),
        "proteins": len(proteins),
    }
    print(f"[{name}] {counts}")
    return counts


def _datapoint_ids(frame: pd.DataFrame, per_residue: bool) -> pd.Series:
    """``"<row_index>"``, or ``"<row_index>:<position>"`` for a per-residue benchmark."""
    rows = frame["row_index"].astype(str)
    if per_residue:
        return (rows + ":" + frame["position"].astype(str)).astype("string")
    return rows.astype("string")


def protein_splits(name: str, rows: list[dict], split_dir: Path) -> pd.DataFrame:
    """Every protein one benchmark measures, with the Lo-Hi split it was given.

    Read from the split's own node file rather than re-derived from the rows, so a
    protein the k-cut assigned stays listed even when none of its rows survived
    (an interaction whose partner landed elsewhere), and one it gave up reads
    ``excluded``.

    Parameters
    ----------
    name : str
        Benchmark name.
    rows : list[dict]
        The benchmark's rows, to name bindingdb's proteins, whose plug key is
        the sequence itself.
    split_dir : Path
        The Lo-Hi split directory.

    Returns
    -------
    pd.DataFrame
        ``node`` (plug's sequence hash, the same across benchmarks), ``protein``
        (the benchmark's own id(s) for it, space-separated), ``split``.

    """
    with gzip.open(split_dir / "nodes" / "proteins.csv.gz", "rt", newline="") as handle:
        nodes = [line for line in csv.DictReader(handle) if line["dataset"] == name]
    if name == "bindingdb":
        accessions: dict[str, set[str]] = {}
        for row in rows:
            if row["uniprot"]:
                accessions.setdefault(node_of(row["sequence"]), set()).add(row["uniprot"])
        named = {node: " ".join(sorted(ids)) for node, ids in accessions.items()}
    else:
        prefix = f"{name}::"
        named = {
            line["node"]: " ".join(
                entity[len(prefix):]
                for entity in line["entities"].split()
                if entity.startswith(prefix)
            )
            for line in nodes
        }
    return pd.DataFrame(
        {
            "node": [line["node"] for line in nodes],
            "protein": [named.get(line["node"], "") for line in nodes],
            "split": [line["split"] for line in nodes],
        }
    ).sort_values("node", ignore_index=True)


def write_card(public: Path) -> None:
    """Write the public repo's dataset card from the manifests already in ``public``.

    Without a card the Hub's viewer concatenates every manifest into one table and calls
    it "train". The ``configs`` here give it one subset per benchmark, whose splits are
    the per-split files, and one more subset for the protein lists.
    """
    names = [name for name in BENCHMARK_NAMES if (public / name / "manifest.parquet").exists()]
    yaml = [
        "---",
        "pretty_name: PLUG challenge — Lo-Hi ids and splits",
        "license: other",
        "tags: [biology, proteins, benchmark, data-leakage]",
        "configs:",
    ]
    for i, name in enumerate(names):
        yaml.append(f"- config_name: {name}")
        if i == 0:
            yaml.append("  default: true")
        yaml.append("  data_files:")
        for filename in SPLIT_FILES:
            yaml += [f"  - split: {filename}", f"    path: {name}/{filename}.parquet"]
    yaml += ["- config_name: proteins", "  data_files:"]
    for name in names:
        yaml += [f"  - split: {name.replace('-', '_')}", f"    path: {name}/proteins.csv"]
    yaml.append("---")

    table = [
        "| Benchmark | Identifier columns | lohi_train | lohi_val | lohi_test | excluded |",
        "|---|---|---:|---:|---:|---:|",
    ]
    for name in names:
        counts = pd.read_parquet(public / name / "manifest.parquet", columns=[SPLIT_COLUMN])[
            SPLIT_COLUMN
        ].value_counts()
        ids = ", ".join(f"`{c}`" for c in get(name).manifest_columns)
        table.append(
            f"| `{name}` | {ids} | "
            + " | ".join(f"{int(counts.get(v, 0)):,}" for v in SPLIT_FILES.values())
            + " |"
        )

    body = f"""
# PLUG challenge — Lo-Hi ids and splits

The public data of the [PLUG challenge]({SPACE_URL}): for each of {len(names)} protein
benchmarks, every datapoint's identifier and the Lo-Hi split it was given.
**There are no labels here** — see [Labels](#labels).

## Layout

One subset per benchmark. Its splits are the Lo-Hi splits:

- `lohi_train`, `lohi_val`: train and tune on these (supervised track).
- `lohi_test`: the held-out rows the supervised track is scored on.
- `excluded`: rows the Lo-Hi k-cut dropped to keep similar proteins out of different
  splits. The unsupervised track scores every row, these included.

{chr(10).join(table)}

Each file has `datapoint_id`, `row_index`, the benchmark's own identifier columns, and
`split`. `<benchmark>/manifest.parquet` holds all of a benchmark's rows in one file, with
`split` empty for an excluded row. The `proteins` subset lists each benchmark's proteins
(`node`, plug's sequence hash, shared across benchmarks) with the Lo-Hi split each was
assigned; `excluded` there marks a protein the k-cut gave up.

## The `datapoint_id`

A row's position in the benchmark as plug's own loader yields it
(`plug.benchmarks.<Benchmark>.rows()`), so `"1234"` is the 1235th row. The per-residue
benchmarks (`allobench`, `passerrank`, `atlas`) add the 1-based residue: `"1234:56"`.
Submissions are keyed on it.

## Labels

This repository holds **only identifiers and split assignments**. Not every benchmark's
licence allows its measurements to be redistributed (LP-PDBBind's does not), so no label,
sequence, mutation or ligand is published here. Load the rows, labels included, from the
original sources with plug's loaders: `row_index` is the row's position there. Each
benchmark's data stays under its original licence.

## Submitting

See the [challenge Space]({SPACE_URL}) for the submission format, the two tracks and how
submissions are scored.
"""
    (public / "README.md").write_text("\n".join(yaml) + "\n" + body)


def main(argv: list[str] | None = None) -> int:
    """Build the challenge data for the requested benchmarks.

    Parameters
    ----------
    argv : list[str] | None
        Argument list; ``None`` reads ``sys.argv``.

    Returns
    -------
    int
        Process exit status.

    """
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument(
        "benchmarks",
        nargs="*",
        default=list(BENCHMARK_NAMES),
        help=f"Benchmarks to build (default: all of {', '.join(BENCHMARK_NAMES)})",
    )
    parser.add_argument("--plug", type=Path, required=True, help="plug checkout")
    parser.add_argument(
        "--split",
        type=Path,
        required=True,
        help="Lo-Hi split directory (holds splits/, nodes/ and provenance.json)",
    )
    parser.add_argument(
        "--out",
        type=Path,
        default=Path(".challenge-data"),
        help="Output directory (default: .challenge-data, which is gitignored)",
    )
    args = parser.parse_args(argv)

    repo_root = Path(__file__).resolve().parent.parent
    summary = {}
    for name in args.benchmarks:
        if name not in BENCHMARK_NAMES:
            print(f"skipping unknown benchmark {name!r}")
            continue
        try:
            summary[name] = build(name, args.plug, args.split, args.out, repo_root)
        except Exception as exc:  # keep going: one benchmark's missing data is not fatal
            print(f"[{name}] FAILED: {type(exc).__name__}: {exc}")

    provenance = json.loads((args.split / "provenance.json").read_text())
    record = {
        "split": f"{args.split.parent.name}/{args.split.name}",
        "files": provenance.get("files", {}),
    }
    (repo_root / "data" / "lohi_provenance.json").write_text(json.dumps(record, indent=2) + "\n")
    write_card(args.out / "public")

    print("\n== summary ==")
    for name, counts in summary.items():
        print(f"{name:12s} {counts}")
    print(
        f"\nPublic ids and splits: {args.out / 'public'}\n"
        f"Private labels: {args.out / 'private'}\n"
        "Next: upload each to its dataset repo — see docs/SETUP.md."
    )
    return 0 if len(summary) == len(args.benchmarks) else 1


if __name__ == "__main__":
    sys.exit(main())