alanclayrichard's picture
score bindingdb on only exact == measurements, accept partial submissions (single or few function only) with unsubmitted set to chance)
580f4e5
Raw History Blame Contribute Delete
9.42 kB
"""Command line entry points for the PLUG challenge.
Participant-facing::
plug-challenge template ./my_submission --track supervised # an empty bundle skeleton
plug-challenge validate my_submission.zip --track supervised # check it before uploading
Organiser-facing::
plug-challenge score my_submission.zip --track supervised --ground-truth ./gt
plug-challenge rebuild # leaderboard <- scores/**
``validate`` needs only the public manifests, so a participant can run it
offline against a downloaded copy of the data repo (``--data-dir``) or let it
fetch the manifests from the Hub.
"""
import argparse
import json
import sys
from pathlib import Path
import pandas as pd
from loguru import logger
from . import aggregate, scoring, store
from .benchmarks import BENCHMARKS, ID_COLUMN, TRACKS, select_track
from .validation import validate_bundle
def _add_track(parser: argparse.ArgumentParser) -> None:
"""Add the ``--track`` argument every participant-facing command takes."""
parser.add_argument(
"--track",
choices=TRACKS,
required=True,
help="supervised: the Lo-Hi test rows; unsupervised: every row, unsplit",
)
def _add_common(parser: argparse.ArgumentParser) -> None:
"""Add the arguments shared by ``validate`` and ``score``."""
parser.add_argument("bundle", type=Path, help="The submission .zip to check")
_add_track(parser)
parser.add_argument(
"--data-dir",
type=Path,
default=None,
help="Directory holding <benchmark>/manifest.parquet (default: fetch from the Hub)",
)
parser.add_argument(
"--require-complete",
action="store_true",
help="Fail on a bundle missing any benchmark (the Space accepts partial bundles, "
"and counts each missing benchmark as chance)",
)
def build_parser() -> argparse.ArgumentParser:
"""Build the argument parser.
Returns
-------
argparse.ArgumentParser
The configured parser with one subcommand per entry point.
"""
parser = argparse.ArgumentParser(
prog="plug-challenge",
description="Build, check and score PLUG challenge submissions.",
epilog=(
"typical use:\n"
" plug-challenge template ./my_submission --track supervised\n"
" cd my_submission && zip ../plug_submission.zip *.csv && cd ..\n"
" plug-challenge validate plug_submission.zip --track supervised\n"
),
formatter_class=argparse.RawDescriptionHelpFormatter,
)
sub = parser.add_subparsers(dest="command", required=True)
_add_common(sub.add_parser("validate", help="Validate a submission bundle"))
score = sub.add_parser("score", help="Validate and score a bundle (organisers)")
_add_common(score)
score.add_argument(
"--ground-truth",
type=Path,
required=True,
help="Directory holding <benchmark>.parquet ground-truth files, every row with its split",
)
score.add_argument(
"--bootstrap",
type=int,
default=0,
help="Bootstrap resamples per metric (0 = point estimates only)",
)
sub.add_parser("rebuild", help="Rebuild the leaderboard file from scores/** (organisers)")
template = sub.add_parser("template", help="Write an empty submission bundle skeleton")
template.add_argument(
"out_dir", type=Path, help="Directory to write the per-benchmark CSVs into"
)
_add_track(template)
template.add_argument(
"--data-dir",
type=Path,
default=None,
help="Directory holding the manifests (default: fetch from the Hub)",
)
return parser
def _expected_ids(track: str, data_dir: Path | None) -> dict[str, set[str]]:
"""Load one track's per-benchmark id sets from the manifests."""
ids = store.manifest_ids(track, local_dir=data_dir)
if not ids:
logger.error(
"No manifests found. Pass --data-dir pointing at a local copy of "
"the challenge data repo, or check network access to the Hub."
)
raise SystemExit(2)
return ids
def command_validate(args: argparse.Namespace) -> int:
"""Validate a bundle and print the report.
Parameters
----------
args : argparse.Namespace
Parsed arguments.
Returns
-------
int
0 if the bundle is valid, 1 otherwise.
"""
result = validate_bundle(
args.bundle,
expected_ids=_expected_ids(args.track, args.data_dir),
require_complete=args.require_complete,
)
if result.ok:
print(f"OK — {len(result.frames)} benchmark file(s) validated.")
if result.warnings:
print("\nWarnings:")
print(result.report(limit=100))
return 0
print("Bundle is not valid:")
print(result.report(limit=100))
return 1
def command_score(args: argparse.Namespace) -> int:
"""Validate, score and print one bundle's metrics.
Parameters
----------
args : argparse.Namespace
Parsed arguments.
Returns
-------
int
0 if the bundle scored, 1 if validation failed.
"""
truth: dict[str, pd.DataFrame] = {}
for spec in BENCHMARKS:
path = args.ground_truth / f"{spec.name}.parquet"
if path.exists():
frame = pd.read_parquet(path)
frame[ID_COLUMN] = frame[ID_COLUMN].astype(str)
truth[spec.name] = select_track(frame, args.track)
else:
logger.warning("No ground truth for {} at {}", spec.name, path)
result = validate_bundle(
args.bundle,
expected_ids={name: set(frame[ID_COLUMN]) for name, frame in truth.items()},
require_complete=args.require_complete,
)
if not result.ok:
print("Bundle is not valid:")
print(result.report(limit=100))
return 1
if result.warnings:
print("Warnings:")
print(result.report(limit=100))
print()
scores = scoring.score_bundle(
result.frames, truth, BENCHMARKS, bootstrap_samples=args.bootstrap
)
print(json.dumps(scores, indent=2))
print(f"\nPLUG Score: {aggregate.plug_score(scores):.4f}")
return 0
def command_rebuild(_: argparse.Namespace) -> int:
"""Rebuild the leaderboard file from the per-submission score files.
Parameters
----------
_ : argparse.Namespace
Unused.
Returns
-------
int
0 on success, 2 when no token is configured.
"""
if not store.configured():
logger.error("HF_TOKEN is not set — cannot write to the private repo.")
return 2
entries = store.all_scores()
frame = pd.DataFrame(entries) if entries else pd.DataFrame()
store.write_leaderboard(frame)
print(f"Rebuilt the leaderboard from {len(entries)} score file(s).")
return 0
def command_template(args: argparse.Namespace) -> int:
"""Write an empty per-benchmark CSV skeleton with the right ids and columns.
Every file comes out with the track's ``datapoint_id`` values already in
place, the manifest's ``row_index`` (and ``position``) beside them to join
against plug's loaders, and the prediction columns filled with zeros, so a
participant only has to overwrite the prediction column(s). cafa5 is the
exception: it is long form, so its file is written with the header alone.
Parameters
----------
args : argparse.Namespace
Parsed arguments.
Returns
-------
int
0 on success.
"""
args.out_dir.mkdir(parents=True, exist_ok=True)
for spec in BENCHMARKS:
try:
manifest = select_track(
store.load_manifest(spec.name, local_dir=args.data_dir), args.track
)
except FileNotFoundError:
logger.warning("Skipping {} — no manifest available.", spec.name)
continue
path = args.out_dir / spec.filename
if spec.long_form:
pd.DataFrame(columns=list(spec.required_columns)).to_csv(path, index=False)
print(
f"wrote {path} (header only — add one row per (protein, branch, GO term) "
f"you predict, for any of the {len(manifest)} proteins in this track)"
)
else:
keys = [ID_COLUMN, "row_index", *(["position"] if spec.per_residue else [])]
frame = manifest[keys].copy()
for column in spec.pred_columns:
frame[column] = 0.0
frame.to_csv(path, index=False)
print(f"wrote {path} ({len(manifest)} rows — replace the zeros)")
print(
f"\nFill in the prediction columns, then zip the directory's contents:\n"
f" cd {args.out_dir} && zip ../plug_submission.zip *.csv"
)
return 0
def main(argv: list[str] | None = None) -> int:
"""Parse arguments and dispatch.
Parameters
----------
argv : list[str] | None
Argument list; ``None`` reads ``sys.argv``.
Returns
-------
int
Process exit status.
"""
args = build_parser().parse_args(argv)
handlers = {
"validate": command_validate,
"score": command_score,
"rebuild": command_rebuild,
"template": command_template,
}
return handlers[args.command](args)
if __name__ == "__main__":
sys.exit(main())