Spaces:
Sleeping
Sleeping
Download theme_generation/cli.py from pachet/theme_generator: direct link, hf CLI and curl.
- Browser
- Download file 14.1 kB
-
https://huggingface.co/spaces/pachet/theme_generator/resolve/main/theme_generation/cli.py
- Command line
-
hf download hf://spaces/pachet/theme_generator/theme_generation/cli.py
-
curl -L -o cli.py https://huggingface.co/spaces/pachet/theme_generator/resolve/main/theme_generation/cli.py
14.1 kB
| """Command-line entry points for generation engines.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import platform | |
| from importlib.metadata import PackageNotFoundError, version | |
| from dataclasses import asdict, replace | |
| from pathlib import Path | |
| from .common import duration_options, normalize_output_key | |
| from .corpus import PREPROCESSING_VERSION, database_digest, training_digest, load_corpus, sequence_endpoint_priors, symbol_stats | |
| from .engines.markov import generate_markov | |
| from .engines.transformer import ( | |
| TransformerConfig, | |
| generate_transformer, | |
| load_transformer_checkpoint, | |
| sample_transformer_checkpoint, | |
| train_and_save_checkpoint, | |
| ) | |
| from .io import write_samples | |
| from .reports import format_allowed_durations, write_generation_report | |
| def add_common_args(parser: argparse.ArgumentParser) -> None: | |
| parser.add_argument("--db", type=Path, default=Path("audit/themes_audit.sqlite")) | |
| parser.add_argument("--output-dir", type=Path) | |
| parser.add_argument("--length", type=int, default=24) | |
| parser.add_argument("--samples", type=int, default=12) | |
| parser.add_argument("--key", default="C") | |
| parser.add_argument("--endpoint-strength", type=float, default=1.0) | |
| parser.add_argument("--seed", type=int, default=42) | |
| parser.add_argument("--split-manifest", type=Path, help="Train only on the train partition of this validated split") | |
| parser.add_argument( | |
| "--min-duration", | |
| default="16th", | |
| help="Shortest generated/training duration label.", | |
| ) | |
| parser.add_argument( | |
| "--duration-grid", | |
| default="16th", | |
| help="Require generated/training durations to be multiples of this value.", | |
| ) | |
| parser.add_argument( | |
| "--no-triplets", | |
| action="store_true", | |
| help="Exclude regular triplet durations from the generated/training vocabulary.", | |
| ) | |
| parser.add_argument( | |
| "--loose-triplets", | |
| action="store_true", | |
| help="Allow triplet durations outside complete beat-aligned groups.", | |
| ) | |
| parser.add_argument("--write-abc", action="store_true", help="Also write ABC files next to the MIDIs.") | |
| parser.add_argument( | |
| "--write-musicxml", | |
| action="store_true", | |
| help="Also write MusicXML files next to the MIDIs.", | |
| ) | |
| def load_generation_inputs(args: argparse.Namespace): | |
| allowed_durations = duration_options(args.min_duration, args.duration_grid, not args.no_triplets) | |
| if not allowed_durations: | |
| raise ValueError(f"No allowed durations remain for min duration {args.min_duration!r}") | |
| records, rejected = load_corpus(args.db, allowed_durations, min_len=6) | |
| if getattr(args, "split_manifest", None): | |
| from .evaluation import validate_manifest | |
| manifest = json.loads(args.split_manifest.read_text()) | |
| validate_manifest(manifest, records, args.db, allowed_durations) | |
| training_ids = set(manifest["partitions"]["train"]) | |
| records = [record for record in records if record.theme_id in training_ids] | |
| sequences = [record.sequence for record in records] | |
| if not sequences: | |
| raise ValueError("No training sequences matched the selected duration and length settings") | |
| stats = symbol_stats(sequences) | |
| stats.update(preprocessing=PREPROCESSING_VERSION, rejected_themes=len(rejected), | |
| database_sha256=database_digest(args.db), corpus_sha256=training_digest(args.db), theme_ids=[record.theme_id for record in records]) | |
| return allowed_durations, sequences, stats, sequence_endpoint_priors(sequences) | |
| def write_run_manifest(args, stats, generated, effective_config=None): | |
| dependencies = {} | |
| for name in ("torch", "vo-regular-bp", "muses", "numpy", "mido"): | |
| try: | |
| dependencies[name] = version(name) | |
| except PackageNotFoundError: | |
| dependencies[name] = "not installed" | |
| checkpoint = getattr(args, "load_checkpoint", None) or getattr(args, "save_checkpoint", None) | |
| payload = {"format": "theme-generation-v1", "preprocessing": PREPROCESSING_VERSION, | |
| "database_sha256": stats["database_sha256"], "training_theme_ids": stats["theme_ids"], | |
| "corpus_sha256": stats["corpus_sha256"], | |
| "python": platform.python_version(), "platform": platform.platform(), "dependencies": dependencies, | |
| "checkpoint_sha256": database_digest(checkpoint) if checkpoint else None, | |
| "effective_model_config": effective_config, | |
| "settings": {key: str(value) if isinstance(value, Path) else value | |
| for key, value in vars(args).items() if key != "func"}, | |
| "sequences": [[(symbol.rpc, symbol.duration) for symbol in sequence] for sequence in generated]} | |
| args.output_dir.mkdir(parents=True, exist_ok=True) | |
| (args.output_dir / "generation.json").write_text(json.dumps(payload, indent=2) + "\n") | |
| def base_settings(args: argparse.Namespace, stats: dict, allowed_durations: set[str]) -> dict[str, object]: | |
| return { | |
| "sequences": stats["sequence_count"], | |
| "events": stats["event_count"], | |
| "vocabulary size": stats["vocab_size"], | |
| "generated event length (including rests)": args.length, | |
| "samples": args.samples, | |
| "output key": args.key, | |
| "minimum duration": args.min_duration, | |
| "duration grid": args.duration_grid, | |
| "triplets allowed": not args.no_triplets, | |
| "triplets grouped": not args.loose_triplets, | |
| "allowed durations": format_allowed_durations(allowed_durations), | |
| "endpoint strength": args.endpoint_strength, | |
| "seed": args.seed, | |
| "preprocessing": stats["preprocessing"], | |
| "rejected themes": stats["rejected_themes"], | |
| "database sha256": stats["database_sha256"], | |
| "corpus sha256": stats["corpus_sha256"], | |
| "rest events": stats["rest_count"], | |
| } | |
| def run_markov(args: argparse.Namespace) -> None: | |
| allowed_durations, sequences, stats, priors = load_generation_inputs(args) | |
| start_weights, end_weights = priors | |
| generated, diagnostics = generate_markov( | |
| sequences=sequences, | |
| length=args.length, | |
| samples=args.samples, | |
| max_order=args.max_order, | |
| start_weights=start_weights, | |
| end_weights=end_weights, | |
| endpoint_strength=args.endpoint_strength, | |
| enforce_triplet_groups=not args.loose_triplets, | |
| seed=args.seed, | |
| ) | |
| write_run_manifest(args, stats, generated) | |
| write_samples( | |
| generated, | |
| output_dir=args.output_dir, | |
| key_name=args.key, | |
| engine_name="vo_regular baseline", | |
| write_abc=args.write_abc, | |
| write_musicxml_files=args.write_musicxml, | |
| ) | |
| settings = base_settings(args, stats, allowed_durations) | |
| settings["max order"] = args.max_order | |
| settings.update(diagnostics) | |
| write_generation_report( | |
| output_dir=args.output_dir, | |
| title="VO-Regular Baseline Generation", | |
| description="This is the key-relative variable-order Markov baseline.", | |
| settings=settings, | |
| stats=stats, | |
| generated=generated, | |
| write_abc=args.write_abc, | |
| write_musicxml=args.write_musicxml, | |
| ) | |
| print(f"Wrote {args.output_dir}") | |
| def run_transformer(args: argparse.Namespace) -> None: | |
| cfg = TransformerConfig( | |
| block_size=args.block_size, | |
| d_model=args.d_model, | |
| nhead=args.nhead, | |
| num_layers=args.layers, | |
| dim_feedforward=args.feedforward, | |
| dropout=args.dropout, | |
| batch_size=args.batch_size, | |
| steps=args.steps, | |
| learning_rate=args.learning_rate, | |
| temperature=args.temperature, | |
| top_k=args.top_k, | |
| max_retries=args.max_retries, | |
| ) | |
| allowed_durations, sequences, stats, priors = load_generation_inputs(args) | |
| start_weights, end_weights = priors | |
| if args.load_checkpoint: | |
| checkpoint = load_transformer_checkpoint(args.load_checkpoint, requested_device=args.device) | |
| if (checkpoint.diagnostics.get("training_theme_ids") != stats["theme_ids"] | |
| or checkpoint.diagnostics.get("corpus_sha256") != stats["corpus_sha256"] | |
| or checkpoint.diagnostics.get("preprocessing") != PREPROCESSING_VERSION): | |
| raise ValueError("Checkpoint training provenance differs or is missing; retrain with this corpus/split before CLI evaluation") | |
| cfg = replace(checkpoint.cfg, temperature=args.temperature, top_k=args.top_k, max_retries=args.max_retries) | |
| generated, diagnostics = sample_transformer_checkpoint( | |
| checkpoint=checkpoint, | |
| length=args.length, | |
| samples=args.samples, | |
| start_weights=start_weights, | |
| end_weights=end_weights, | |
| endpoint_strength=args.endpoint_strength, | |
| enforce_triplet_groups=not args.loose_triplets, | |
| seed=args.seed, | |
| temperature=args.temperature, | |
| top_k=args.top_k, | |
| max_retries=args.max_retries, | |
| allowed_durations=allowed_durations, | |
| ) | |
| elif args.save_checkpoint: | |
| checkpoint = train_and_save_checkpoint( | |
| sequences=sequences, | |
| cfg=cfg, | |
| seed=args.seed, | |
| requested_device=args.device, | |
| path=args.save_checkpoint, | |
| provenance={"training_theme_ids": stats["theme_ids"], "database_sha256": stats["database_sha256"], "corpus_sha256": stats["corpus_sha256"], "preprocessing": PREPROCESSING_VERSION, "seed": args.seed}, | |
| ) | |
| generated, diagnostics = sample_transformer_checkpoint( | |
| checkpoint=checkpoint, | |
| length=args.length, | |
| samples=args.samples, | |
| start_weights=start_weights, | |
| end_weights=end_weights, | |
| endpoint_strength=args.endpoint_strength, | |
| enforce_triplet_groups=not args.loose_triplets, | |
| seed=args.seed, | |
| temperature=args.temperature, | |
| top_k=args.top_k, | |
| max_retries=args.max_retries, | |
| allowed_durations=allowed_durations, | |
| ) | |
| diagnostics["saved checkpoint"] = str(args.save_checkpoint) | |
| else: | |
| generated, diagnostics = generate_transformer( | |
| sequences=sequences, | |
| length=args.length, | |
| samples=args.samples, | |
| start_weights=start_weights, | |
| end_weights=end_weights, | |
| endpoint_strength=args.endpoint_strength, | |
| enforce_triplet_groups=not args.loose_triplets, | |
| seed=args.seed, | |
| cfg=cfg, | |
| device=args.device, | |
| ) | |
| write_run_manifest(args, stats, generated, effective_config=asdict(cfg)) | |
| write_samples( | |
| generated, | |
| output_dir=args.output_dir, | |
| key_name=args.key, | |
| engine_name="transformer baseline", | |
| write_abc=args.write_abc, | |
| write_musicxml_files=args.write_musicxml, | |
| ) | |
| settings = base_settings(args, stats, allowed_durations) | |
| settings.update( | |
| { | |
| "block size": cfg.block_size, | |
| "d model": cfg.d_model, | |
| "heads": cfg.nhead, | |
| "layers": cfg.num_layers, | |
| "feedforward": cfg.dim_feedforward, | |
| "dropout": cfg.dropout, | |
| "batch size": cfg.batch_size, | |
| "learning rate": cfg.learning_rate, | |
| "temperature": cfg.temperature, | |
| "top k": cfg.top_k, | |
| } | |
| ) | |
| settings.update(diagnostics) | |
| write_generation_report( | |
| output_dir=args.output_dir, | |
| title="Transformer Baseline Generation", | |
| description="This is the first key-relative tiny transformer baseline.", | |
| settings=settings, | |
| stats=stats, | |
| generated=generated, | |
| write_abc=args.write_abc, | |
| write_musicxml=args.write_musicxml, | |
| ) | |
| print(f"Wrote {args.output_dir}") | |
| def build_parser() -> argparse.ArgumentParser: | |
| parser = argparse.ArgumentParser(description="Generate short key-relative theme samples.") | |
| subparsers = parser.add_subparsers(dest="engine", required=True) | |
| markov = subparsers.add_parser("markov", help="Run the vo_regular variable-order Markov engine.") | |
| add_common_args(markov) | |
| markov.set_defaults(output_dir=Path("outputs/vo_regular_baseline"), func=run_markov) | |
| markov.add_argument("--max-order", type=int, default=4) | |
| transformer = subparsers.add_parser("transformer", help="Run the tiny PyTorch transformer engine.") | |
| add_common_args(transformer) | |
| transformer.set_defaults(output_dir=Path("outputs/transformer_baseline"), func=run_transformer) | |
| transformer.add_argument("--block-size", type=int, default=64) | |
| transformer.add_argument("--d-model", type=int, default=96) | |
| transformer.add_argument("--nhead", type=int, default=4) | |
| transformer.add_argument("--layers", type=int, default=3) | |
| transformer.add_argument("--feedforward", type=int, default=192) | |
| transformer.add_argument("--dropout", type=float, default=0.1) | |
| transformer.add_argument("--batch-size", type=int, default=64) | |
| transformer.add_argument("--steps", type=int, default=800) | |
| transformer.add_argument("--learning-rate", type=float, default=3e-4) | |
| transformer.add_argument("--temperature", type=float, default=1.0) | |
| transformer.add_argument("--top-k", type=int, default=16) | |
| transformer.add_argument("--max-retries", type=int, default=100) | |
| transformer.add_argument("--device", default="auto", help="auto, cpu, cuda, or mps.") | |
| transformer.add_argument("--save-checkpoint", type=Path, help="Train once and save a reusable checkpoint.") | |
| transformer.add_argument("--load-checkpoint", type=Path, help="Generate from a saved checkpoint instead of training.") | |
| return parser | |
| def main(argv: list[str] | None = None) -> None: | |
| parser = build_parser() | |
| args = parser.parse_args(argv) | |
| try: | |
| args.key = normalize_output_key(args.key) | |
| args.func(args) | |
| except (RuntimeError, ValueError) as exc: | |
| parser.exit(1, f"error: {exc}\n") | |
| if __name__ == "__main__": | |
| main() | |