pachet's picture
Deploy corrected Theme Lab (d1780324)
1ebc6d4 verified
Raw History Blame Contribute Delete
3.35 kB
"""Prepare blinded baseline listening pairs; never synthesize human ratings."""
import argparse
import hashlib
import json
from pathlib import Path
import random
import shutil
def prepare_listening_pairs(pilot: Path, output: Path, seed: int = 20261006):
results = json.loads((pilot / "results.json").read_text())
pairs = []
for run in results["runs"]:
root = pilot / f"seed_{run['seed']}"
engines = {engine: json.loads((root / engine / "generation.json").read_text())
for engine in ("markov", "transformer")}
if any(payload["corpus_sha256"] != results["corpus_sha256"] for payload in engines.values()):
raise ValueError("Listening stimuli do not match the pilot corpus")
count = len(engines["markov"]["sequences"])
if count != len(engines["transformer"]["sequences"]):
raise ValueError("Listening conditions must have equal sample counts")
for index in range(1, count+1):
pairs.append((run["seed"], index, root))
if output.exists():
raise ValueError("Study directory already exists; preserve existing ratings and choose a new directory")
rng = random.Random(seed)
rng.shuffle(pairs)
# Equal A/B assignments across the complete panel; participant order is
# randomized separately by the study administrator.
assignments = [i % 2 for i in range(len(pairs))]
rng.shuffle(assignments)
trials, key = [], []
for number, ((model_seed, index, root), flipped) in enumerate(zip(pairs, assignments), 1):
conditions = ("transformer", "markov") if flipped else ("markov", "transformer")
trial = {"trial_id": f"trial_{number:03d}", "a": None, "b": None,
"musical_quality_a": None, "musical_quality_b": None, "preference": None, "notes": ""}
hidden = {"trial_id": trial["trial_id"], "model_seed": model_seed, "sample_index": index}
for side, condition in zip(("a", "b"), conditions):
source = root / condition / f"generated_{index:02d}.mid"
target = output / "stimuli" / f"{trial['trial_id']}_{side}.mid"
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(source, target)
trial[side] = target.relative_to(output).as_posix()
hidden[side] = {"condition": condition, "sha256": hashlib.sha256(target.read_bytes()).hexdigest()}
trials.append(trial)
key.append(hidden)
metadata = {"format": "theme-listening-trials-v1", "status": "unrated", "seed": seed,
"corpus_sha256": results["corpus_sha256"], "trials": trials}
(output / "trials.json").write_text(json.dumps(metadata, indent=2)+"\n")
(output / "administrator_key.json").write_text(json.dumps(key, indent=2)+"\n")
return metadata
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--pilot", type=Path, default=Path("outputs/october_pilot"))
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--seed", type=int, default=20261006)
args = parser.parse_args()
payload = prepare_listening_pairs(args.pilot, args.output, args.seed)
print(f"Prepared {len(payload['trials'])} unrated pairs; keep administrator_key.json hidden from listeners")
if __name__ == "__main__":
main()