| { |
| "suite": "slm-benchmark-v1.1", |
| "generated_at": "Illustrative data — not an evaluation", |
| "models": [ |
| { |
| "id": "demo-110m", "name": "Illustrative Base 110M", "params_m": 110, |
| "track": "base", "complete": true, "rank_range": [1, 2], |
| "composite": {"score": 16.8, "ci": [15.1, 18.6], "classification": "reliably above chance"}, |
| "benchmarks": { |
| "hellaswag": {"score": 12.4, "ci": [10.7, 14.2]}, |
| "piqa": {"score": 20.6, "ci": [17.1, 24.0]}, |
| "arc_easy": {"score": 15.0, "ci": [11.5, 18.5]}, |
| "arc_challenge": {"score": 7.1, "ci": [3.4, 10.7]}, |
| "commonsense_qa": {"score": 8.9, "ci": [5.2, 12.6]}, |
| "blimp": {"score": 27.2, "ci": [24.8, 29.6]} |
| } |
| }, |
| { |
| "id": "demo-160m", "name": "Illustrative Base 160M", "params_m": 160, |
| "track": "base", "complete": true, "rank_range": [1, 2], |
| "composite": {"score": 17.7, "ci": [16.0, 19.5], "classification": "reliably above chance"}, |
| "benchmarks": { |
| "hellaswag": {"score": 13.9, "ci": [12.1, 15.6]}, |
| "piqa": {"score": 22.1, "ci": [18.7, 25.4]}, |
| "arc_easy": {"score": 16.3, "ci": [12.9, 19.7]}, |
| "arc_challenge": {"score": 9.8, "ci": [6.1, 13.5]}, |
| "commonsense_qa": {"score": 10.6, "ci": [7.1, 14.0]}, |
| "blimp": {"score": 25.8, "ci": [23.5, 28.1]} |
| } |
| }, |
| { |
| "id": "demo-60m", "name": "Illustrative Base 60M", "params_m": 60, |
| "track": "base", "complete": true, "rank_range": [3, 3], |
| "composite": {"score": 5.6, "ci": [3.9, 7.4], "classification": "reliably above chance"}, |
| "benchmarks": { |
| "hellaswag": {"score": 2.1, "ci": [0.5, 3.8]}, |
| "piqa": {"score": 7.3, "ci": [4.0, 10.5]}, |
| "arc_easy": {"score": 3.8, "ci": [0.8, 6.7]}, |
| "arc_challenge": {"score": 1.4, "ci": [-2.4, 5.2]}, |
| "commonsense_qa": {"score": 1.2, "ci": [-2.1, 4.4]}, |
| "blimp": {"score": 13.6, "ci": [11.2, 16.0]} |
| } |
| } |
| ] |
| } |
|
|
