Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
score bindingdb on only exact == measurements, accept partial submissions (single or few function only) with unsubmitted set to chance)
580f4e5 Download plug_challenge/cli.py from PrismScienceOrg/PLUG-challenge: direct link, hf CLI and curl.
- Browser
- Download file 9.42 kB
-
https://huggingface.co/spaces/PrismScienceOrg/PLUG-challenge/resolve/main/plug_challenge/cli.py
- Command line
-
hf download hf://spaces/PrismScienceOrg/PLUG-challenge/plug_challenge/cli.py
-
curl -L -o cli.py https://huggingface.co/spaces/PrismScienceOrg/PLUG-challenge/resolve/main/plug_challenge/cli.py
9.42 kB
| """Command line entry points for the PLUG challenge. | |
| Participant-facing:: | |
| plug-challenge template ./my_submission --track supervised # an empty bundle skeleton | |
| plug-challenge validate my_submission.zip --track supervised # check it before uploading | |
| Organiser-facing:: | |
| plug-challenge score my_submission.zip --track supervised --ground-truth ./gt | |
| plug-challenge rebuild # leaderboard <- scores/** | |
| ``validate`` needs only the public manifests, so a participant can run it | |
| offline against a downloaded copy of the data repo (``--data-dir``) or let it | |
| fetch the manifests from the Hub. | |
| """ | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import pandas as pd | |
| from loguru import logger | |
| from . import aggregate, scoring, store | |
| from .benchmarks import BENCHMARKS, ID_COLUMN, TRACKS, select_track | |
| from .validation import validate_bundle | |
| def _add_track(parser: argparse.ArgumentParser) -> None: | |
| """Add the ``--track`` argument every participant-facing command takes.""" | |
| parser.add_argument( | |
| "--track", | |
| choices=TRACKS, | |
| required=True, | |
| help="supervised: the Lo-Hi test rows; unsupervised: every row, unsplit", | |
| ) | |
| def _add_common(parser: argparse.ArgumentParser) -> None: | |
| """Add the arguments shared by ``validate`` and ``score``.""" | |
| parser.add_argument("bundle", type=Path, help="The submission .zip to check") | |
| _add_track(parser) | |
| parser.add_argument( | |
| "--data-dir", | |
| type=Path, | |
| default=None, | |
| help="Directory holding <benchmark>/manifest.parquet (default: fetch from the Hub)", | |
| ) | |
| parser.add_argument( | |
| "--require-complete", | |
| action="store_true", | |
| help="Fail on a bundle missing any benchmark (the Space accepts partial bundles, " | |
| "and counts each missing benchmark as chance)", | |
| ) | |
| def build_parser() -> argparse.ArgumentParser: | |
| """Build the argument parser. | |
| Returns | |
| ------- | |
| argparse.ArgumentParser | |
| The configured parser with one subcommand per entry point. | |
| """ | |
| parser = argparse.ArgumentParser( | |
| prog="plug-challenge", | |
| description="Build, check and score PLUG challenge submissions.", | |
| epilog=( | |
| "typical use:\n" | |
| " plug-challenge template ./my_submission --track supervised\n" | |
| " cd my_submission && zip ../plug_submission.zip *.csv && cd ..\n" | |
| " plug-challenge validate plug_submission.zip --track supervised\n" | |
| ), | |
| formatter_class=argparse.RawDescriptionHelpFormatter, | |
| ) | |
| sub = parser.add_subparsers(dest="command", required=True) | |
| _add_common(sub.add_parser("validate", help="Validate a submission bundle")) | |
| score = sub.add_parser("score", help="Validate and score a bundle (organisers)") | |
| _add_common(score) | |
| score.add_argument( | |
| "--ground-truth", | |
| type=Path, | |
| required=True, | |
| help="Directory holding <benchmark>.parquet ground-truth files, every row with its split", | |
| ) | |
| score.add_argument( | |
| "--bootstrap", | |
| type=int, | |
| default=0, | |
| help="Bootstrap resamples per metric (0 = point estimates only)", | |
| ) | |
| sub.add_parser("rebuild", help="Rebuild the leaderboard file from scores/** (organisers)") | |
| template = sub.add_parser("template", help="Write an empty submission bundle skeleton") | |
| template.add_argument( | |
| "out_dir", type=Path, help="Directory to write the per-benchmark CSVs into" | |
| ) | |
| _add_track(template) | |
| template.add_argument( | |
| "--data-dir", | |
| type=Path, | |
| default=None, | |
| help="Directory holding the manifests (default: fetch from the Hub)", | |
| ) | |
| return parser | |
| def _expected_ids(track: str, data_dir: Path | None) -> dict[str, set[str]]: | |
| """Load one track's per-benchmark id sets from the manifests.""" | |
| ids = store.manifest_ids(track, local_dir=data_dir) | |
| if not ids: | |
| logger.error( | |
| "No manifests found. Pass --data-dir pointing at a local copy of " | |
| "the challenge data repo, or check network access to the Hub." | |
| ) | |
| raise SystemExit(2) | |
| return ids | |
| def command_validate(args: argparse.Namespace) -> int: | |
| """Validate a bundle and print the report. | |
| Parameters | |
| ---------- | |
| args : argparse.Namespace | |
| Parsed arguments. | |
| Returns | |
| ------- | |
| int | |
| 0 if the bundle is valid, 1 otherwise. | |
| """ | |
| result = validate_bundle( | |
| args.bundle, | |
| expected_ids=_expected_ids(args.track, args.data_dir), | |
| require_complete=args.require_complete, | |
| ) | |
| if result.ok: | |
| print(f"OK — {len(result.frames)} benchmark file(s) validated.") | |
| if result.warnings: | |
| print("\nWarnings:") | |
| print(result.report(limit=100)) | |
| return 0 | |
| print("Bundle is not valid:") | |
| print(result.report(limit=100)) | |
| return 1 | |
| def command_score(args: argparse.Namespace) -> int: | |
| """Validate, score and print one bundle's metrics. | |
| Parameters | |
| ---------- | |
| args : argparse.Namespace | |
| Parsed arguments. | |
| Returns | |
| ------- | |
| int | |
| 0 if the bundle scored, 1 if validation failed. | |
| """ | |
| truth: dict[str, pd.DataFrame] = {} | |
| for spec in BENCHMARKS: | |
| path = args.ground_truth / f"{spec.name}.parquet" | |
| if path.exists(): | |
| frame = pd.read_parquet(path) | |
| frame[ID_COLUMN] = frame[ID_COLUMN].astype(str) | |
| truth[spec.name] = select_track(frame, args.track) | |
| else: | |
| logger.warning("No ground truth for {} at {}", spec.name, path) | |
| result = validate_bundle( | |
| args.bundle, | |
| expected_ids={name: set(frame[ID_COLUMN]) for name, frame in truth.items()}, | |
| require_complete=args.require_complete, | |
| ) | |
| if not result.ok: | |
| print("Bundle is not valid:") | |
| print(result.report(limit=100)) | |
| return 1 | |
| if result.warnings: | |
| print("Warnings:") | |
| print(result.report(limit=100)) | |
| print() | |
| scores = scoring.score_bundle( | |
| result.frames, truth, BENCHMARKS, bootstrap_samples=args.bootstrap | |
| ) | |
| print(json.dumps(scores, indent=2)) | |
| print(f"\nPLUG Score: {aggregate.plug_score(scores):.4f}") | |
| return 0 | |
| def command_rebuild(_: argparse.Namespace) -> int: | |
| """Rebuild the leaderboard file from the per-submission score files. | |
| Parameters | |
| ---------- | |
| _ : argparse.Namespace | |
| Unused. | |
| Returns | |
| ------- | |
| int | |
| 0 on success, 2 when no token is configured. | |
| """ | |
| if not store.configured(): | |
| logger.error("HF_TOKEN is not set — cannot write to the private repo.") | |
| return 2 | |
| entries = store.all_scores() | |
| frame = pd.DataFrame(entries) if entries else pd.DataFrame() | |
| store.write_leaderboard(frame) | |
| print(f"Rebuilt the leaderboard from {len(entries)} score file(s).") | |
| return 0 | |
| def command_template(args: argparse.Namespace) -> int: | |
| """Write an empty per-benchmark CSV skeleton with the right ids and columns. | |
| Every file comes out with the track's ``datapoint_id`` values already in | |
| place, the manifest's ``row_index`` (and ``position``) beside them to join | |
| against plug's loaders, and the prediction columns filled with zeros, so a | |
| participant only has to overwrite the prediction column(s). cafa5 is the | |
| exception: it is long form, so its file is written with the header alone. | |
| Parameters | |
| ---------- | |
| args : argparse.Namespace | |
| Parsed arguments. | |
| Returns | |
| ------- | |
| int | |
| 0 on success. | |
| """ | |
| args.out_dir.mkdir(parents=True, exist_ok=True) | |
| for spec in BENCHMARKS: | |
| try: | |
| manifest = select_track( | |
| store.load_manifest(spec.name, local_dir=args.data_dir), args.track | |
| ) | |
| except FileNotFoundError: | |
| logger.warning("Skipping {} — no manifest available.", spec.name) | |
| continue | |
| path = args.out_dir / spec.filename | |
| if spec.long_form: | |
| pd.DataFrame(columns=list(spec.required_columns)).to_csv(path, index=False) | |
| print( | |
| f"wrote {path} (header only — add one row per (protein, branch, GO term) " | |
| f"you predict, for any of the {len(manifest)} proteins in this track)" | |
| ) | |
| else: | |
| keys = [ID_COLUMN, "row_index", *(["position"] if spec.per_residue else [])] | |
| frame = manifest[keys].copy() | |
| for column in spec.pred_columns: | |
| frame[column] = 0.0 | |
| frame.to_csv(path, index=False) | |
| print(f"wrote {path} ({len(manifest)} rows — replace the zeros)") | |
| print( | |
| f"\nFill in the prediction columns, then zip the directory's contents:\n" | |
| f" cd {args.out_dir} && zip ../plug_submission.zip *.csv" | |
| ) | |
| return 0 | |
| def main(argv: list[str] | None = None) -> int: | |
| """Parse arguments and dispatch. | |
| Parameters | |
| ---------- | |
| argv : list[str] | None | |
| Argument list; ``None`` reads ``sys.argv``. | |
| Returns | |
| ------- | |
| int | |
| Process exit status. | |
| """ | |
| args = build_parser().parse_args(argv) | |
| handlers = { | |
| "validate": command_validate, | |
| "score": command_score, | |
| "rebuild": command_rebuild, | |
| "template": command_template, | |
| } | |
| return handlers[args.command](args) | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |