pachet's picture
Deploy corrected Theme Lab (d1780324)
1ebc6d4 verified
Raw History Blame Contribute Delete
14.1 kB
"""Command-line entry points for generation engines."""
from __future__ import annotations
import argparse
import json
import platform
from importlib.metadata import PackageNotFoundError, version
from dataclasses import asdict, replace
from pathlib import Path
from .common import duration_options, normalize_output_key
from .corpus import PREPROCESSING_VERSION, database_digest, training_digest, load_corpus, sequence_endpoint_priors, symbol_stats
from .engines.markov import generate_markov
from .engines.transformer import (
TransformerConfig,
generate_transformer,
load_transformer_checkpoint,
sample_transformer_checkpoint,
train_and_save_checkpoint,
)
from .io import write_samples
from .reports import format_allowed_durations, write_generation_report
def add_common_args(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--db", type=Path, default=Path("audit/themes_audit.sqlite"))
parser.add_argument("--output-dir", type=Path)
parser.add_argument("--length", type=int, default=24)
parser.add_argument("--samples", type=int, default=12)
parser.add_argument("--key", default="C")
parser.add_argument("--endpoint-strength", type=float, default=1.0)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--split-manifest", type=Path, help="Train only on the train partition of this validated split")
parser.add_argument(
"--min-duration",
default="16th",
help="Shortest generated/training duration label.",
)
parser.add_argument(
"--duration-grid",
default="16th",
help="Require generated/training durations to be multiples of this value.",
)
parser.add_argument(
"--no-triplets",
action="store_true",
help="Exclude regular triplet durations from the generated/training vocabulary.",
)
parser.add_argument(
"--loose-triplets",
action="store_true",
help="Allow triplet durations outside complete beat-aligned groups.",
)
parser.add_argument("--write-abc", action="store_true", help="Also write ABC files next to the MIDIs.")
parser.add_argument(
"--write-musicxml",
action="store_true",
help="Also write MusicXML files next to the MIDIs.",
)
def load_generation_inputs(args: argparse.Namespace):
allowed_durations = duration_options(args.min_duration, args.duration_grid, not args.no_triplets)
if not allowed_durations:
raise ValueError(f"No allowed durations remain for min duration {args.min_duration!r}")
records, rejected = load_corpus(args.db, allowed_durations, min_len=6)
if getattr(args, "split_manifest", None):
from .evaluation import validate_manifest
manifest = json.loads(args.split_manifest.read_text())
validate_manifest(manifest, records, args.db, allowed_durations)
training_ids = set(manifest["partitions"]["train"])
records = [record for record in records if record.theme_id in training_ids]
sequences = [record.sequence for record in records]
if not sequences:
raise ValueError("No training sequences matched the selected duration and length settings")
stats = symbol_stats(sequences)
stats.update(preprocessing=PREPROCESSING_VERSION, rejected_themes=len(rejected),
database_sha256=database_digest(args.db), corpus_sha256=training_digest(args.db), theme_ids=[record.theme_id for record in records])
return allowed_durations, sequences, stats, sequence_endpoint_priors(sequences)
def write_run_manifest(args, stats, generated, effective_config=None):
dependencies = {}
for name in ("torch", "vo-regular-bp", "muses", "numpy", "mido"):
try:
dependencies[name] = version(name)
except PackageNotFoundError:
dependencies[name] = "not installed"
checkpoint = getattr(args, "load_checkpoint", None) or getattr(args, "save_checkpoint", None)
payload = {"format": "theme-generation-v1", "preprocessing": PREPROCESSING_VERSION,
"database_sha256": stats["database_sha256"], "training_theme_ids": stats["theme_ids"],
"corpus_sha256": stats["corpus_sha256"],
"python": platform.python_version(), "platform": platform.platform(), "dependencies": dependencies,
"checkpoint_sha256": database_digest(checkpoint) if checkpoint else None,
"effective_model_config": effective_config,
"settings": {key: str(value) if isinstance(value, Path) else value
for key, value in vars(args).items() if key != "func"},
"sequences": [[(symbol.rpc, symbol.duration) for symbol in sequence] for sequence in generated]}
args.output_dir.mkdir(parents=True, exist_ok=True)
(args.output_dir / "generation.json").write_text(json.dumps(payload, indent=2) + "\n")
def base_settings(args: argparse.Namespace, stats: dict, allowed_durations: set[str]) -> dict[str, object]:
return {
"sequences": stats["sequence_count"],
"events": stats["event_count"],
"vocabulary size": stats["vocab_size"],
"generated event length (including rests)": args.length,
"samples": args.samples,
"output key": args.key,
"minimum duration": args.min_duration,
"duration grid": args.duration_grid,
"triplets allowed": not args.no_triplets,
"triplets grouped": not args.loose_triplets,
"allowed durations": format_allowed_durations(allowed_durations),
"endpoint strength": args.endpoint_strength,
"seed": args.seed,
"preprocessing": stats["preprocessing"],
"rejected themes": stats["rejected_themes"],
"database sha256": stats["database_sha256"],
"corpus sha256": stats["corpus_sha256"],
"rest events": stats["rest_count"],
}
def run_markov(args: argparse.Namespace) -> None:
allowed_durations, sequences, stats, priors = load_generation_inputs(args)
start_weights, end_weights = priors
generated, diagnostics = generate_markov(
sequences=sequences,
length=args.length,
samples=args.samples,
max_order=args.max_order,
start_weights=start_weights,
end_weights=end_weights,
endpoint_strength=args.endpoint_strength,
enforce_triplet_groups=not args.loose_triplets,
seed=args.seed,
)
write_run_manifest(args, stats, generated)
write_samples(
generated,
output_dir=args.output_dir,
key_name=args.key,
engine_name="vo_regular baseline",
write_abc=args.write_abc,
write_musicxml_files=args.write_musicxml,
)
settings = base_settings(args, stats, allowed_durations)
settings["max order"] = args.max_order
settings.update(diagnostics)
write_generation_report(
output_dir=args.output_dir,
title="VO-Regular Baseline Generation",
description="This is the key-relative variable-order Markov baseline.",
settings=settings,
stats=stats,
generated=generated,
write_abc=args.write_abc,
write_musicxml=args.write_musicxml,
)
print(f"Wrote {args.output_dir}")
def run_transformer(args: argparse.Namespace) -> None:
cfg = TransformerConfig(
block_size=args.block_size,
d_model=args.d_model,
nhead=args.nhead,
num_layers=args.layers,
dim_feedforward=args.feedforward,
dropout=args.dropout,
batch_size=args.batch_size,
steps=args.steps,
learning_rate=args.learning_rate,
temperature=args.temperature,
top_k=args.top_k,
max_retries=args.max_retries,
)
allowed_durations, sequences, stats, priors = load_generation_inputs(args)
start_weights, end_weights = priors
if args.load_checkpoint:
checkpoint = load_transformer_checkpoint(args.load_checkpoint, requested_device=args.device)
if (checkpoint.diagnostics.get("training_theme_ids") != stats["theme_ids"]
or checkpoint.diagnostics.get("corpus_sha256") != stats["corpus_sha256"]
or checkpoint.diagnostics.get("preprocessing") != PREPROCESSING_VERSION):
raise ValueError("Checkpoint training provenance differs or is missing; retrain with this corpus/split before CLI evaluation")
cfg = replace(checkpoint.cfg, temperature=args.temperature, top_k=args.top_k, max_retries=args.max_retries)
generated, diagnostics = sample_transformer_checkpoint(
checkpoint=checkpoint,
length=args.length,
samples=args.samples,
start_weights=start_weights,
end_weights=end_weights,
endpoint_strength=args.endpoint_strength,
enforce_triplet_groups=not args.loose_triplets,
seed=args.seed,
temperature=args.temperature,
top_k=args.top_k,
max_retries=args.max_retries,
allowed_durations=allowed_durations,
)
elif args.save_checkpoint:
checkpoint = train_and_save_checkpoint(
sequences=sequences,
cfg=cfg,
seed=args.seed,
requested_device=args.device,
path=args.save_checkpoint,
provenance={"training_theme_ids": stats["theme_ids"], "database_sha256": stats["database_sha256"], "corpus_sha256": stats["corpus_sha256"], "preprocessing": PREPROCESSING_VERSION, "seed": args.seed},
)
generated, diagnostics = sample_transformer_checkpoint(
checkpoint=checkpoint,
length=args.length,
samples=args.samples,
start_weights=start_weights,
end_weights=end_weights,
endpoint_strength=args.endpoint_strength,
enforce_triplet_groups=not args.loose_triplets,
seed=args.seed,
temperature=args.temperature,
top_k=args.top_k,
max_retries=args.max_retries,
allowed_durations=allowed_durations,
)
diagnostics["saved checkpoint"] = str(args.save_checkpoint)
else:
generated, diagnostics = generate_transformer(
sequences=sequences,
length=args.length,
samples=args.samples,
start_weights=start_weights,
end_weights=end_weights,
endpoint_strength=args.endpoint_strength,
enforce_triplet_groups=not args.loose_triplets,
seed=args.seed,
cfg=cfg,
device=args.device,
)
write_run_manifest(args, stats, generated, effective_config=asdict(cfg))
write_samples(
generated,
output_dir=args.output_dir,
key_name=args.key,
engine_name="transformer baseline",
write_abc=args.write_abc,
write_musicxml_files=args.write_musicxml,
)
settings = base_settings(args, stats, allowed_durations)
settings.update(
{
"block size": cfg.block_size,
"d model": cfg.d_model,
"heads": cfg.nhead,
"layers": cfg.num_layers,
"feedforward": cfg.dim_feedforward,
"dropout": cfg.dropout,
"batch size": cfg.batch_size,
"learning rate": cfg.learning_rate,
"temperature": cfg.temperature,
"top k": cfg.top_k,
}
)
settings.update(diagnostics)
write_generation_report(
output_dir=args.output_dir,
title="Transformer Baseline Generation",
description="This is the first key-relative tiny transformer baseline.",
settings=settings,
stats=stats,
generated=generated,
write_abc=args.write_abc,
write_musicxml=args.write_musicxml,
)
print(f"Wrote {args.output_dir}")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Generate short key-relative theme samples.")
subparsers = parser.add_subparsers(dest="engine", required=True)
markov = subparsers.add_parser("markov", help="Run the vo_regular variable-order Markov engine.")
add_common_args(markov)
markov.set_defaults(output_dir=Path("outputs/vo_regular_baseline"), func=run_markov)
markov.add_argument("--max-order", type=int, default=4)
transformer = subparsers.add_parser("transformer", help="Run the tiny PyTorch transformer engine.")
add_common_args(transformer)
transformer.set_defaults(output_dir=Path("outputs/transformer_baseline"), func=run_transformer)
transformer.add_argument("--block-size", type=int, default=64)
transformer.add_argument("--d-model", type=int, default=96)
transformer.add_argument("--nhead", type=int, default=4)
transformer.add_argument("--layers", type=int, default=3)
transformer.add_argument("--feedforward", type=int, default=192)
transformer.add_argument("--dropout", type=float, default=0.1)
transformer.add_argument("--batch-size", type=int, default=64)
transformer.add_argument("--steps", type=int, default=800)
transformer.add_argument("--learning-rate", type=float, default=3e-4)
transformer.add_argument("--temperature", type=float, default=1.0)
transformer.add_argument("--top-k", type=int, default=16)
transformer.add_argument("--max-retries", type=int, default=100)
transformer.add_argument("--device", default="auto", help="auto, cpu, cuda, or mps.")
transformer.add_argument("--save-checkpoint", type=Path, help="Train once and save a reusable checkpoint.")
transformer.add_argument("--load-checkpoint", type=Path, help="Generate from a saved checkpoint instead of training.")
return parser
def main(argv: list[str] | None = None) -> None:
parser = build_parser()
args = parser.parse_args(argv)
try:
args.key = normalize_output_key(args.key)
args.func(args)
except (RuntimeError, ValueError) as exc:
parser.exit(1, f"error: {exc}\n")
if __name__ == "__main__":
main()