File size: 6,190 Bytes
04436f5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | #!/usr/bin/env python3
"""Render the model-card figures from saved scores; matplotlib 3.10.8."""
import json
from pathlib import Path
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.ticker import PercentFormatter
import numpy as np
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'docs/figures'
COLORS = {'Solstice-1': '#176b65', 'OpenSML-150M': '#67768a',
'SmolLM2-135M Instruct': '#c58a37', 'Checkpoint 167': '#9476a7'}
def read(path):
return json.loads((ROOT / path).read_text())
def save(fig, name):
OUT.mkdir(parents=True, exist_ok=True)
for ext in ('png', 'svg'):
fig.savefig(OUT / f'{name}.{ext}', dpi=180, facecolor='white')
plt.close(fig)
def main():
plt.rcParams.update({'font.family': 'DejaVu Sans', 'font.size': 11,
'axes.spines.top': False, 'axes.spines.right': False,
'axes.spines.left': False, 'axes.titleweight': 'bold'})
choice = read('results/public/choices/results.json')['results']
public = read('results/baselines/published_models.json')['models']
old = public['OpenSML/OpenSML-150M']
smol = public['HuggingFaceTB/SmolLM2-135M-Instruct']
fig, axes = plt.subplots(2, 2, figsize=(11.5, 7.6))
fig.subplots_adjust(left=.2, right=.97, top=.84, bottom=.16, wspace=.85, hspace=.55)
fig.suptitle('Solstice-1 | Knowledge and commonsense', x=.08, ha='left', y=.97,
fontsize=19, fontweight='bold')
fig.text(.08, .91, 'Zero-shot normalized accuracy 路 higher is better', fontsize=12)
for ax, (key, label) in zip(axes.flat, [('arc_easy', 'ARC Easy'),
('arc_challenge', 'ARC Challenge'), ('piqa', 'PIQA'), ('hellaswag', 'HellaSwag')]):
names = ['Solstice-1', 'OpenSML-150M']
values = [choice[key]['acc_norm'], old['choices'][key]['acc_norm']]
if key in smol['choices']:
names.append('SmolLM2-135M Instruct')
values.append(smol['choices'][key]['acc_norm'])
ax.barh(range(len(names)), values, height=.55, color=[COLORS[n] for n in names])
ax.set_yticks(range(len(names)), [n.replace(' Instruct', '\nInstruct') for n in names], fontsize=9)
ax.invert_yaxis(); ax.set_xlim(0, .8); ax.set_xticks([0, .2, .4, .6, .8])
ax.xaxis.set_major_formatter(PercentFormatter(1, decimals=0))
ax.set_title(label, loc='left', pad=10); ax.grid(axis='x', alpha=.15); ax.set_axisbelow(True)
ax.tick_params(axis='y', length=0)
for y, value in enumerate(values):
ax.text(value+.015, y, f'{value:.2%}' if names[y] != 'SmolLM2-135M Instruct' else f'{value:.1%}',
va='center', fontsize=10)
fig.text(.08, .06, 'Baseline values are publisher reports; evaluators and prompts differ.\n'
'SmolLM2 publishes ARC only as an average (37.3%); its separate ARC scores are not inferred.',
fontsize=10, color='#4b5563')
save(fig, 'benchmark-comparison')
own = read('results/public/ifeval/results.json')['metrics']
prior = read('results/baselines/167/ifeval/results.json')['metrics']
keys = [('strict', 'prompt'), ('strict', 'instruction'), ('loose', 'prompt'), ('loose', 'instruction')]
fig, ax = plt.subplots(figsize=(11.5, 5.9))
fig.subplots_adjust(left=.08, right=.98, top=.8, bottom=.24)
fig.suptitle('Solstice-1 | Instruction following', x=.08, ha='left', y=.97,
fontsize=19, fontweight='bold')
fig.text(.08, .9, 'IFEval accuracy 路 higher is better', fontsize=12)
x = np.arange(4)
for i, (name, data) in enumerate([('OpenSML-150M', old['ifeval']),
('Checkpoint 167', prior), ('Solstice-1', own)]):
values = [data[k][u+'_accuracy'] for k, u in keys]
bars = ax.bar(x+(i-1)*.25, values, width=.22, label=name, color=COLORS[name])
ax.bar_label(bars, labels=[f'{v:.2%}' for v in values], padding=4, fontsize=9)
ax.set_xticks(x, ['Strict\nprompts', 'Strict\ninstructions', 'Loose\nprompts', 'Loose\ninstructions'])
ax.set_ylim(0, .4); ax.set_yticks([0, .1, .2, .3, .4]); ax.yaxis.set_major_formatter(PercentFormatter(1, decimals=0))
ax.grid(axis='y', alpha=.15); ax.set_axisbelow(True)
ax.legend(loc='upper left', bbox_to_anchor=(0, -.18), ncols=3, frameon=False)
fig.text(.08, .035, '150M used different formatting and a longer output cap. SmolLM2 reports an aggregate (29.9%),\n'
'so its strict/loose sub-scores are not plotted. No normalized-accuracy metric applies to IFEval.',
fontsize=10, color='#4b5563')
save(fig, 'ifeval-comparison')
chat = read('results/public/chat/results.json')
categories = [('reading', 'Reading supplied text'), ('conversation', 'Conversation context'),
('knowledge', 'Factual knowledge'), ('instructions', 'Instructions'),
('answerability', 'Answerability'), ('reasoning', 'Short reasoning')]
values = [chat['categories'][key]['answer_match']/chat['categories'][key]['count'] for key, _ in categories]
fig, ax = plt.subplots(figsize=(11.5, 5.8))
fig.subplots_adjust(left=.25, right=.96, top=.79, bottom=.2)
fig.suptitle('Solstice-1 | Short-chat diagnostics', x=.08, ha='left', y=.97,
fontsize=19, fontweight='bold')
fig.text(.08, .9, 'Conservative automatic answer matching 路 authored panel', fontsize=12)
ax.barh(range(6), values, color=COLORS['Solstice-1'], height=.6)
ax.set_yticks(range(6), [label for _, label in categories]); ax.invert_yaxis()
ax.set_xlim(0, 1); ax.xaxis.set_major_formatter(PercentFormatter(1, decimals=0))
ax.tick_params(axis='y', length=0); ax.grid(axis='x', alpha=.15); ax.set_axisbelow(True)
for y, value in enumerate(values): ax.text(value+.015, y, f'{value:.1%}', va='center')
fig.text(.08, .05, '30 checked responses per category. This is not a broad human assessment.\n'
'Natural stopping is 99.48% across all 192 responses; stopping correctly does not imply a correct answer.',
fontsize=10, color='#4b5563')
save(fig, 'chat-diagnostics')
if __name__ == '__main__':
main()
|