"""Command line entry points for the PLUG challenge. Participant-facing:: plug-challenge template ./my_submission --track supervised # an empty bundle skeleton plug-challenge validate my_submission.zip --track supervised # check it before uploading Organiser-facing:: plug-challenge score my_submission.zip --track supervised --ground-truth ./gt plug-challenge rebuild # leaderboard <- scores/** ``validate`` needs only the public manifests, so a participant can run it offline against a downloaded copy of the data repo (``--data-dir``) or let it fetch the manifests from the Hub. """ import argparse import json import sys from pathlib import Path import pandas as pd from loguru import logger from . import aggregate, scoring, store from .benchmarks import BENCHMARKS, ID_COLUMN, TRACKS, select_track from .validation import validate_bundle def _add_track(parser: argparse.ArgumentParser) -> None: """Add the ``--track`` argument every participant-facing command takes.""" parser.add_argument( "--track", choices=TRACKS, required=True, help="supervised: the Lo-Hi test rows; unsupervised: every row, unsplit", ) def _add_common(parser: argparse.ArgumentParser) -> None: """Add the arguments shared by ``validate`` and ``score``.""" parser.add_argument("bundle", type=Path, help="The submission .zip to check") _add_track(parser) parser.add_argument( "--data-dir", type=Path, default=None, help="Directory holding /manifest.parquet (default: fetch from the Hub)", ) parser.add_argument( "--require-complete", action="store_true", help="Fail on a bundle missing any benchmark (the Space accepts partial bundles, " "and counts each missing benchmark as chance)", ) def build_parser() -> argparse.ArgumentParser: """Build the argument parser. Returns ------- argparse.ArgumentParser The configured parser with one subcommand per entry point. """ parser = argparse.ArgumentParser( prog="plug-challenge", description="Build, check and score PLUG challenge submissions.", epilog=( "typical use:\n" " plug-challenge template ./my_submission --track supervised\n" " cd my_submission && zip ../plug_submission.zip *.csv && cd ..\n" " plug-challenge validate plug_submission.zip --track supervised\n" ), formatter_class=argparse.RawDescriptionHelpFormatter, ) sub = parser.add_subparsers(dest="command", required=True) _add_common(sub.add_parser("validate", help="Validate a submission bundle")) score = sub.add_parser("score", help="Validate and score a bundle (organisers)") _add_common(score) score.add_argument( "--ground-truth", type=Path, required=True, help="Directory holding .parquet ground-truth files, every row with its split", ) score.add_argument( "--bootstrap", type=int, default=0, help="Bootstrap resamples per metric (0 = point estimates only)", ) sub.add_parser("rebuild", help="Rebuild the leaderboard file from scores/** (organisers)") template = sub.add_parser("template", help="Write an empty submission bundle skeleton") template.add_argument( "out_dir", type=Path, help="Directory to write the per-benchmark CSVs into" ) _add_track(template) template.add_argument( "--data-dir", type=Path, default=None, help="Directory holding the manifests (default: fetch from the Hub)", ) return parser def _expected_ids(track: str, data_dir: Path | None) -> dict[str, set[str]]: """Load one track's per-benchmark id sets from the manifests.""" ids = store.manifest_ids(track, local_dir=data_dir) if not ids: logger.error( "No manifests found. Pass --data-dir pointing at a local copy of " "the challenge data repo, or check network access to the Hub." ) raise SystemExit(2) return ids def command_validate(args: argparse.Namespace) -> int: """Validate a bundle and print the report. Parameters ---------- args : argparse.Namespace Parsed arguments. Returns ------- int 0 if the bundle is valid, 1 otherwise. """ result = validate_bundle( args.bundle, expected_ids=_expected_ids(args.track, args.data_dir), require_complete=args.require_complete, ) if result.ok: print(f"OK — {len(result.frames)} benchmark file(s) validated.") if result.warnings: print("\nWarnings:") print(result.report(limit=100)) return 0 print("Bundle is not valid:") print(result.report(limit=100)) return 1 def command_score(args: argparse.Namespace) -> int: """Validate, score and print one bundle's metrics. Parameters ---------- args : argparse.Namespace Parsed arguments. Returns ------- int 0 if the bundle scored, 1 if validation failed. """ truth: dict[str, pd.DataFrame] = {} for spec in BENCHMARKS: path = args.ground_truth / f"{spec.name}.parquet" if path.exists(): frame = pd.read_parquet(path) frame[ID_COLUMN] = frame[ID_COLUMN].astype(str) truth[spec.name] = select_track(frame, args.track) else: logger.warning("No ground truth for {} at {}", spec.name, path) result = validate_bundle( args.bundle, expected_ids={name: set(frame[ID_COLUMN]) for name, frame in truth.items()}, require_complete=args.require_complete, ) if not result.ok: print("Bundle is not valid:") print(result.report(limit=100)) return 1 if result.warnings: print("Warnings:") print(result.report(limit=100)) print() scores = scoring.score_bundle( result.frames, truth, BENCHMARKS, bootstrap_samples=args.bootstrap ) print(json.dumps(scores, indent=2)) print(f"\nPLUG Score: {aggregate.plug_score(scores):.4f}") return 0 def command_rebuild(_: argparse.Namespace) -> int: """Rebuild the leaderboard file from the per-submission score files. Parameters ---------- _ : argparse.Namespace Unused. Returns ------- int 0 on success, 2 when no token is configured. """ if not store.configured(): logger.error("HF_TOKEN is not set — cannot write to the private repo.") return 2 entries = store.all_scores() frame = pd.DataFrame(entries) if entries else pd.DataFrame() store.write_leaderboard(frame) print(f"Rebuilt the leaderboard from {len(entries)} score file(s).") return 0 def command_template(args: argparse.Namespace) -> int: """Write an empty per-benchmark CSV skeleton with the right ids and columns. Every file comes out with the track's ``datapoint_id`` values already in place, the manifest's ``row_index`` (and ``position``) beside them to join against plug's loaders, and the prediction columns filled with zeros, so a participant only has to overwrite the prediction column(s). cafa5 is the exception: it is long form, so its file is written with the header alone. Parameters ---------- args : argparse.Namespace Parsed arguments. Returns ------- int 0 on success. """ args.out_dir.mkdir(parents=True, exist_ok=True) for spec in BENCHMARKS: try: manifest = select_track( store.load_manifest(spec.name, local_dir=args.data_dir), args.track ) except FileNotFoundError: logger.warning("Skipping {} — no manifest available.", spec.name) continue path = args.out_dir / spec.filename if spec.long_form: pd.DataFrame(columns=list(spec.required_columns)).to_csv(path, index=False) print( f"wrote {path} (header only — add one row per (protein, branch, GO term) " f"you predict, for any of the {len(manifest)} proteins in this track)" ) else: keys = [ID_COLUMN, "row_index", *(["position"] if spec.per_residue else [])] frame = manifest[keys].copy() for column in spec.pred_columns: frame[column] = 0.0 frame.to_csv(path, index=False) print(f"wrote {path} ({len(manifest)} rows — replace the zeros)") print( f"\nFill in the prediction columns, then zip the directory's contents:\n" f" cd {args.out_dir} && zip ../plug_submission.zip *.csv" ) return 0 def main(argv: list[str] | None = None) -> int: """Parse arguments and dispatch. Parameters ---------- argv : list[str] | None Argument list; ``None`` reads ``sys.argv``. Returns ------- int Process exit status. """ args = build_parser().parse_args(argv) handlers = { "validate": command_validate, "score": command_score, "rebuild": command_rebuild, "template": command_template, } return handlers[args.command](args) if __name__ == "__main__": sys.exit(main())