Add files using upload-large-folder tool

.gitattributes

@@ -300,3 +300,43 @@ SDAR-100M-ep4-block4-op7-10/composition/autoregressive/hf_model_id_generations.j
 SDAR-100M-ep4-block4-op7-10/composition/autoregressive/hf_model_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
 SDAR-100M-ep4-block4-op9-12/composition/autoregressive/hf_model_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
 Qwen3-100M-ep4/composition/graphs/checkpoint-85276_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-ep4/composition/graphs/checkpoint-85276_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-ep4-block8/composition/autoregressive/checkpoint-86183_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-ep4/composition/checkpoint-85276_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-ep4/composition/checkpoint-85276_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/graphs/checkpoint-39008_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_ood_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/graphs/checkpoint-39470_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum_16_8_4_2/composition/autoregressive/checkpoint-39470_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum_8_4_2_1/composition/autoregressive/checkpoint-39008_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum-1_8/composition/autoregressive/checkpoint-39008_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-midtrain-ar_diff-block_curriculum-2_16/composition/autoregressive/checkpoint-39470_id_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_ood_generations.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-ep1-block4/composition/low_confidence_static/block8/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-ep1-block4/composition/autoregressive/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text
+SDAR-100M-ep1-block4/composition/low_confidence_static/block4/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl filter=lfs diff=lfs merge=lfs -text

Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "results/context/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2", "results_dir": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs", "per_file": [{"file": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 73614, "value_mismatch": 536991, "dependency_mismatch": 698222, "answer_mismatch": 744241, "extra_nodes": 627017, "missing_nodes": 1077262, "perfect_with_extra": 9, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "value_mismatch": 536991, "dependency_mismatch": 698222, "answer_mismatch": 744241, "missing_nodes": 1077262, "extra_nodes": 627017, "perfect": 73614, "perfect_with_extra": 9}}, "results_path": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 959844, "dependency_mismatch": 1020173, "answer_mismatch": 1093236, "extra_nodes": 890642, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "value_mismatch": 959844, "dependency_mismatch": 1020173, "answer_mismatch": 1093236, "extra_nodes": 890642, "missing_nodes": 1280000}}, "results_path": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 73614, "value_mismatch": 1496835, "dependency_mismatch": 1718395, "answer_mismatch": 1837477, "extra_nodes": 1517659, "missing_nodes": 2357262, "perfect_with_extra": 9, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/0.999zoo_op2-20+0.001teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "value_mismatch": 1496835, "dependency_mismatch": 1718395, "answer_mismatch": 1837477, "missing_nodes": 2357262, "extra_nodes": 1517659, "perfect": 73614, "perfect_with_extra": 9}}}}

Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/graphs/results.json

@@ -0,0 +1,910 @@
+{
+  "files": {
+    "hf_model_id_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 0.577,
+              "pass@2": 0.635,
+              "pass@4": 0.701,
+              "pass@8": 0.753,
+              "pass@16": 0.807,
+              "pass@32": 0.835,
+              "pass@64": 0.863,
+              "pass@128": 0.884
+            },
+            "3": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.001,
+              "pass@64": 0.001,
+              "pass@128": 0.001
+            },
+            "4": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "5": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "6": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "7": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "8": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "9": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "10": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 0.577,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@2": {
+              "2": 0.635,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@4": {
+              "2": 0.701,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@8": {
+              "2": 0.753,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@16": {
+              "2": 0.807,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@32": {
+              "2": 0.835,
+              "3": 0.001,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@64": {
+              "2": 0.863,
+              "3": 0.001,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@128": {
+              "2": 0.884,
+              "3": 0.001,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.0641111111111111,
+            "pass@2": 0.07055555555555555,
+            "pass@4": 0.07788888888888888,
+            "pass@8": 0.08366666666666667,
+            "pass@16": 0.08966666666666667,
+            "pass@32": 0.09288888888888888,
+            "pass@64": 0.096,
+            "pass@128": 0.09833333333333333
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "2": {
+            "pass@1": 0.577,
+            "pass@2": 0.635,
+            "pass@4": 0.701,
+            "pass@8": 0.753,
+            "pass@16": 0.807,
+            "pass@32": 0.835,
+            "pass@64": 0.863,
+            "pass@128": 0.884
+          },
+          "3": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.001,
+            "pass@64": 0.001,
+            "pass@128": 0.001
+          },
+          "4": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "5": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "6": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "7": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "8": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "9": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "10": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "2": 0.577,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@2": {
+            "2": 0.635,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@4": {
+            "2": 0.701,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@8": {
+            "2": 0.753,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@16": {
+            "2": 0.807,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@32": {
+            "2": 0.835,
+            "3": 0.001,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@64": {
+            "2": 0.863,
+            "3": 0.001,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@128": {
+            "2": 0.884,
+            "3": 0.001,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.0641111111111111,
+          "pass@2": 0.07055555555555555,
+          "pass@4": 0.07788888888888888,
+          "pass@8": 0.08366666666666667,
+          "pass@16": 0.08966666666666667,
+          "pass@32": 0.09288888888888888,
+          "pass@64": 0.096,
+          "pass@128": 0.09833333333333333
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "teachers_in_school": {
+            "per_op": {
+              "2": 1000,
+              "3": 1000,
+              "4": 1000,
+              "5": 1000,
+              "6": 1000,
+              "7": 1000,
+              "8": 1000,
+              "9": 1000,
+              "10": 1000
+            },
+            "total": 9000
+          }
+        },
+        "total": {
+          "per_op": {
+            "2": 1000,
+            "3": 1000,
+            "4": 1000,
+            "5": 1000,
+            "6": 1000,
+            "7": 1000,
+            "8": 1000,
+            "9": 1000,
+            "10": 1000
+          },
+          "total": 9000
+        }
+      }
+    },
+    "hf_model_ood_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "12": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "13": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "14": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "15": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "16": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "17": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "18": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "19": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "20": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@2": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@4": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@8": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@16": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@32": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@64": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@128": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "12": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "13": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "14": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "15": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "16": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "17": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "18": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "19": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "20": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@2": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@4": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@8": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@16": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@32": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@64": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@128": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.0,
+          "pass@2": 0.0,
+          "pass@4": 0.0,
+          "pass@8": 0.0,
+          "pass@16": 0.0,
+          "pass@32": 0.0,
+          "pass@64": 0.0,
+          "pass@128": 0.0
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "teachers_in_school": {
+            "per_op": {
+              "11": 1000,
+              "12": 1000,
+              "13": 1000,
+              "14": 1000,
+              "15": 1000,
+              "16": 1000,
+              "17": 1000,
+              "18": 1000,
+              "19": 1000,
+              "20": 1000
+            },
+            "total": 10000
+          }
+        },
+        "total": {
+          "per_op": {
+            "11": 1000,
+            "12": 1000,
+            "13": 1000,
+            "14": 1000,
+            "15": 1000,
+            "16": 1000,
+            "17": 1000,
+            "18": 1000,
+            "19": 1000,
+            "20": 1000
+          },
+          "total": 10000
+        }
+      }
+    }
+  }
+}

Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_metrics.json

@@ -0,0 +1,1077 @@
+{
+  "checkpoint": "hf_model",
+  "id": {
+    "answer_accuracy": 0.35395746527777777,
+    "avg_response_len": 143.9383420138889,
+    "avg_loss": 0.0,
+    "count": 1152000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.35395746527777777,
+      "pass@2": 0.36394313210848733,
+      "pass@4": 0.3721463061179857,
+      "pass@8": 0.38002626115669125,
+      "pass@16": 0.3881806229770293,
+      "pass@32": 0.3970353048170402,
+      "pass@64": 0.40672349924466805,
+      "pass@128": 0.41733333333333333
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.179671875,
+        "pass@2": 0.1866203248031496,
+        "pass@4": 0.19349498397075368,
+        "pass@8": 0.20171124100488724,
+        "pass@16": 0.21184839720232201,
+        "pass@32": 0.22407444095023307,
+        "pass@64": 0.23828823396176474,
+        "pass@128": 0.255
+      },
+      "2": {
+        "pass@1": 0.9675390625,
+        "pass@2": 0.9754135088582678,
+        "pass@4": 0.9807076276715406,
+        "pass@8": 0.9842055326995033,
+        "pass@16": 0.9866531838082899,
+        "pass@32": 0.9888710784240634,
+        "pass@64": 0.991123358518923,
+        "pass@128": 0.993
+      },
+      "3": {
+        "pass@1": 0.7970390625,
+        "pass@2": 0.8376731053149615,
+        "pass@4": 0.8661379510686164,
+        "pass@8": 0.8884730983835463,
+        "pass@16": 0.9065882701957276,
+        "pass@32": 0.9221857999677552,
+        "pass@64": 0.9361523489727314,
+        "pass@128": 0.949
+      },
+      "4": {
+        "pass@1": 0.2107890625,
+        "pass@2": 0.21529441437007876,
+        "pass@4": 0.21916330239970003,
+        "pass@8": 0.22346821216105478,
+        "pass@16": 0.22872197474530756,
+        "pass@32": 0.23562179093658991,
+        "pass@64": 0.24510618631947784,
+        "pass@128": 0.258
+      },
+      "5": {
+        "pass@1": 0.2116171875,
+        "pass@2": 0.21723412893700783,
+        "pass@4": 0.22284044347581547,
+        "pass@8": 0.22918635588080552,
+        "pass@16": 0.23692081774417614,
+        "pass@32": 0.24659684355460448,
+        "pass@64": 0.25860912185373264,
+        "pass@128": 0.272
+      },
+      "6": {
+        "pass@1": 0.2580546875,
+        "pass@2": 0.262111343503937,
+        "pass@4": 0.26583126621672293,
+        "pass@8": 0.269561781409488,
+        "pass@16": 0.2735911486266416,
+        "pass@32": 0.2786441597934989,
+        "pass@64": 0.2852150182516317,
+        "pass@128": 0.293
+      },
+      "7": {
+        "pass@1": 0.195703125,
+        "pass@2": 0.2011721210629921,
+        "pass@4": 0.20649363198350204,
+        "pass@8": 0.212816902115117,
+        "pass@16": 0.22024960690112913,
+        "pass@32": 0.2283513294970211,
+        "pass@64": 0.23693507099206224,
+        "pass@128": 0.247
+      },
+      "8": {
+        "pass@1": 0.18925,
+        "pass@2": 0.19675135334645666,
+        "pass@4": 0.20417071484814395,
+        "pass@8": 0.21275267862195865,
+        "pass@16": 0.22278762964396398,
+        "pass@32": 0.23415911788441254,
+        "pass@64": 0.2463119989893093,
+        "pass@128": 0.258
+      },
+      "9": {
+        "pass@1": 0.175953125,
+        "pass@2": 0.18321788877952747,
+        "pass@4": 0.1904768334270717,
+        "pass@8": 0.19806054813384352,
+        "pass@16": 0.20626457792569045,
+        "pass@32": 0.21481318234518765,
+        "pass@64": 0.22277015534238243,
+        "pass@128": 0.231
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.179671875,
+      "2": 0.9675390625,
+      "3": 0.7970390625,
+      "4": 0.2107890625,
+      "5": 0.2116171875,
+      "6": 0.2580546875,
+      "7": 0.195703125,
+      "8": 0.18925,
+      "9": 0.175953125
+    },
+    "per_op_avg_response_len": {
+      "10": 167.241765625,
+      "2": 111.679328125,
+      "3": 128.0244453125,
+      "4": 125.2879921875,
+      "5": 135.821828125,
+      "6": 152.88628125,
+      "7": 142.7650703125,
+      "8": 163.613359375,
+      "9": 168.1250078125
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 167.241765625,
+      "2": 111.679328125,
+      "3": 128.0244453125,
+      "4": 125.2879921875,
+      "5": 135.821828125,
+      "6": 152.88628125,
+      "7": 142.7650703125,
+      "8": 163.613359375,
+      "9": 168.1250078125
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      },
+      "teachers_in_school": {
+        "count": 1152000,
+        "correct": 407759,
+        "answer_accuracy": 0.35395746527777777,
+        "avg_response_len": 143.9383420138889,
+        "resp_tokens_sum": 165816970,
+        "pass_at_k": {
+          "pass@1": 0.35395746527777777,
+          "pass@2": 0.36394313210848733,
+          "pass@4": 0.3721463061179857,
+          "pass@8": 0.38002626115669125,
+          "pass@16": 0.3881806229770293,
+          "pass@32": 0.3970353048170402,
+          "pass@64": 0.40672349924466805,
+          "pass@128": 0.41733333333333333
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.179671875,
+            "pass@2": 0.1866203248031496,
+            "pass@4": 0.19349498397075368,
+            "pass@8": 0.20171124100488724,
+            "pass@16": 0.21184839720232201,
+            "pass@32": 0.22407444095023307,
+            "pass@64": 0.23828823396176474,
+            "pass@128": 0.255
+          },
+          "2": {
+            "pass@1": 0.9675390625,
+            "pass@2": 0.9754135088582678,
+            "pass@4": 0.9807076276715406,
+            "pass@8": 0.9842055326995033,
+            "pass@16": 0.9866531838082899,
+            "pass@32": 0.9888710784240634,
+            "pass@64": 0.991123358518923,
+            "pass@128": 0.993
+          },
+          "3": {
+            "pass@1": 0.7970390625,
+            "pass@2": 0.8376731053149615,
+            "pass@4": 0.8661379510686164,
+            "pass@8": 0.8884730983835463,
+            "pass@16": 0.9065882701957276,
+            "pass@32": 0.9221857999677552,
+            "pass@64": 0.9361523489727314,
+            "pass@128": 0.949
+          },
+          "4": {
+            "pass@1": 0.2107890625,
+            "pass@2": 0.21529441437007876,
+            "pass@4": 0.21916330239970003,
+            "pass@8": 0.22346821216105478,
+            "pass@16": 0.22872197474530756,
+            "pass@32": 0.23562179093658991,
+            "pass@64": 0.24510618631947784,
+            "pass@128": 0.258
+          },
+          "5": {
+            "pass@1": 0.2116171875,
+            "pass@2": 0.21723412893700783,
+            "pass@4": 0.22284044347581547,
+            "pass@8": 0.22918635588080552,
+            "pass@16": 0.23692081774417614,
+            "pass@32": 0.24659684355460448,
+            "pass@64": 0.25860912185373264,
+            "pass@128": 0.272
+          },
+          "6": {
+            "pass@1": 0.2580546875,
+            "pass@2": 0.262111343503937,
+            "pass@4": 0.26583126621672293,
+            "pass@8": 0.269561781409488,
+            "pass@16": 0.2735911486266416,
+            "pass@32": 0.2786441597934989,
+            "pass@64": 0.2852150182516317,
+            "pass@128": 0.293
+          },
+          "7": {
+            "pass@1": 0.195703125,
+            "pass@2": 0.2011721210629921,
+            "pass@4": 0.20649363198350204,
+            "pass@8": 0.212816902115117,
+            "pass@16": 0.22024960690112913,
+            "pass@32": 0.2283513294970211,
+            "pass@64": 0.23693507099206224,
+            "pass@128": 0.247
+          },
+          "8": {
+            "pass@1": 0.18925,
+            "pass@2": 0.19675135334645666,
+            "pass@4": 0.20417071484814395,
+            "pass@8": 0.21275267862195865,
+            "pass@16": 0.22278762964396398,
+            "pass@32": 0.23415911788441254,
+            "pass@64": 0.2463119989893093,
+            "pass@128": 0.258
+          },
+          "9": {
+            "pass@1": 0.175953125,
+            "pass@2": 0.18321788877952747,
+            "pass@4": 0.1904768334270717,
+            "pass@8": 0.19806054813384352,
+            "pass@16": 0.20626457792569045,
+            "pass@32": 0.21481318234518765,
+            "pass@64": 0.22277015534238243,
+            "pass@128": 0.231
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      }
+    }
+  },
+  "ood": {
+    "answer_accuracy": 0.14591171875,
+    "avg_response_len": 202.49695546875,
+    "avg_loss": 0.0,
+    "count": 1280000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.14591171875,
+      "pass@2": 0.1588627337598429,
+      "pass@4": 0.16978443717660308,
+      "pass@8": 0.17994934512615257,
+      "pass@16": 0.1899462114350564,
+      "pass@32": 0.20014317351317526,
+      "pass@64": 0.2107443148217583,
+      "pass@128": 0.2222
+    },
+    "per_op_pass_at_k": {
+      "11": {
+        "pass@1": 0.1763515625,
+        "pass@2": 0.1854534940944882,
+        "pass@4": 0.1933735462129733,
+        "pass@8": 0.20147957136226124,
+        "pass@16": 0.21037463664286407,
+        "pass@32": 0.22037157099835816,
+        "pass@64": 0.23143147890828927,
+        "pass@128": 0.243
+      },
+      "12": {
+        "pass@1": 0.1750078125,
+        "pass@2": 0.18632972440944887,
+        "pass@4": 0.19674250149981243,
+        "pass@8": 0.2075677772984937,
+        "pass@16": 0.21899942949146833,
+        "pass@32": 0.23008881738333872,
+        "pass@64": 0.24004304392698875,
+        "pass@128": 0.249
+      },
+      "13": {
+        "pass@1": 0.14003125,
+        "pass@2": 0.15481865157480312,
+        "pass@4": 0.16692630361829766,
+        "pass@8": 0.17772499892830299,
+        "pass@16": 0.1882665689521189,
+        "pass@32": 0.19883793436883665,
+        "pass@64": 0.20967385171136876,
+        "pass@128": 0.221
+      },
+      "14": {
+        "pass@1": 0.1436328125,
+        "pass@2": 0.15869389763779526,
+        "pass@4": 0.17226924925009368,
+        "pass@8": 0.1845543174527869,
+        "pass@16": 0.19568671047542624,
+        "pass@32": 0.2065402438692653,
+        "pass@64": 0.21755881523205667,
+        "pass@128": 0.229
+      },
+      "15": {
+        "pass@1": 0.1338828125,
+        "pass@2": 0.14767212106299207,
+        "pass@4": 0.1592121984439444,
+        "pass@8": 0.1693664968394039,
+        "pass@16": 0.1790848548813016,
+        "pass@32": 0.1888553870136373,
+        "pass@64": 0.19929553877537814,
+        "pass@128": 0.211
+      },
+      "16": {
+        "pass@1": 0.155671875,
+        "pass@2": 0.16875172244094488,
+        "pass@4": 0.17969398078365192,
+        "pass@8": 0.1898988485117523,
+        "pass@16": 0.1991953499621563,
+        "pass@32": 0.20812386294309929,
+        "pass@64": 0.21791788802494516,
+        "pass@128": 0.23
+      },
+      "17": {
+        "pass@1": 0.14675,
+        "pass@2": 0.16014972933070862,
+        "pass@4": 0.16990317519685033,
+        "pass@8": 0.17901342366976633,
+        "pass@16": 0.1890970246223302,
+        "pass@32": 0.2009162762964177,
+        "pass@64": 0.21379618773979528,
+        "pass@128": 0.227
+      },
+      "18": {
+        "pass@1": 0.123859375,
+        "pass@2": 0.13845374015748027,
+        "pass@4": 0.15128099362579675,
+        "pass@8": 0.1636793233057228,
+        "pass@16": 0.17590328285323045,
+        "pass@32": 0.18769288846945548,
+        "pass@64": 0.19939121996448883,
+        "pass@128": 0.213
+      },
+      "19": {
+        "pass@1": 0.1425859375,
+        "pass@2": 0.1542038631889764,
+        "pass@4": 0.16308084739407575,
+        "pass@8": 0.17024050661671622,
+        "pass@16": 0.17678480815036582,
+        "pass@32": 0.18411748530607766,
+        "pass@64": 0.19279372965274186,
+        "pass@128": 0.203
+      },
+      "20": {
+        "pass@1": 0.12134375,
+        "pass@2": 0.13410039370078738,
+        "pass@4": 0.1453615757405324,
+        "pass@8": 0.1559681872763174,
+        "pass@16": 0.1660694483193041,
+        "pass@32": 0.17588726848327202,
+        "pass@64": 0.1855413942815293,
+        "pass@128": 0.196
+      }
+    },
+    "per_op_accuracy": {
+      "11": 0.1763515625,
+      "12": 0.1750078125,
+      "13": 0.14003125,
+      "14": 0.1436328125,
+      "15": 0.1338828125,
+      "16": 0.155671875,
+      "17": 0.14675,
+      "18": 0.123859375,
+      "19": 0.1425859375,
+      "20": 0.12134375
+    },
+    "per_op_avg_response_len": {
+      "11": 175.3768046875,
+      "12": 176.2021953125,
+      "13": 176.847875,
+      "14": 195.620578125,
+      "15": 203.4362421875,
+      "16": 206.30290625,
+      "17": 220.5351796875,
+      "18": 222.7138984375,
+      "19": 220.836265625,
+      "20": 227.097609375
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "11": 175.3768046875,
+      "12": 176.2021953125,
+      "13": 176.847875,
+      "14": 195.620578125,
+      "15": 203.4362421875,
+      "16": 206.30290625,
+      "17": 220.5351796875,
+      "18": 222.7138984375,
+      "19": 220.836265625,
+      "20": 227.097609375
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      },
+      "teachers_in_school": {
+        "count": 1280000,
+        "correct": 186767,
+        "answer_accuracy": 0.14591171875,
+        "avg_response_len": 202.49695546875,
+        "resp_tokens_sum": 259196103,
+        "pass_at_k": {
+          "pass@1": 0.14591171875,
+          "pass@2": 0.1588627337598429,
+          "pass@4": 0.16978443717660308,
+          "pass@8": 0.17994934512615257,
+          "pass@16": 0.1899462114350564,
+          "pass@32": 0.20014317351317526,
+          "pass@64": 0.2107443148217583,
+          "pass@128": 0.2222
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.1763515625,
+            "pass@2": 0.1854534940944882,
+            "pass@4": 0.1933735462129733,
+            "pass@8": 0.20147957136226124,
+            "pass@16": 0.21037463664286407,
+            "pass@32": 0.22037157099835816,
+            "pass@64": 0.23143147890828927,
+            "pass@128": 0.243
+          },
+          "12": {
+            "pass@1": 0.1750078125,
+            "pass@2": 0.18632972440944887,
+            "pass@4": 0.19674250149981243,
+            "pass@8": 0.2075677772984937,
+            "pass@16": 0.21899942949146833,
+            "pass@32": 0.23008881738333872,
+            "pass@64": 0.24004304392698875,
+            "pass@128": 0.249
+          },
+          "13": {
+            "pass@1": 0.14003125,
+            "pass@2": 0.15481865157480312,
+            "pass@4": 0.16692630361829766,
+            "pass@8": 0.17772499892830299,
+            "pass@16": 0.1882665689521189,
+            "pass@32": 0.19883793436883665,
+            "pass@64": 0.20967385171136876,
+            "pass@128": 0.221
+          },
+          "14": {
+            "pass@1": 0.1436328125,
+            "pass@2": 0.15869389763779526,
+            "pass@4": 0.17226924925009368,
+            "pass@8": 0.1845543174527869,
+            "pass@16": 0.19568671047542624,
+            "pass@32": 0.2065402438692653,
+            "pass@64": 0.21755881523205667,
+            "pass@128": 0.229
+          },
+          "15": {
+            "pass@1": 0.1338828125,
+            "pass@2": 0.14767212106299207,
+            "pass@4": 0.1592121984439444,
+            "pass@8": 0.1693664968394039,
+            "pass@16": 0.1790848548813016,
+            "pass@32": 0.1888553870136373,
+            "pass@64": 0.19929553877537814,
+            "pass@128": 0.211
+          },
+          "16": {
+            "pass@1": 0.155671875,
+            "pass@2": 0.16875172244094488,
+            "pass@4": 0.17969398078365192,
+            "pass@8": 0.1898988485117523,
+            "pass@16": 0.1991953499621563,
+            "pass@32": 0.20812386294309929,
+            "pass@64": 0.21791788802494516,
+            "pass@128": 0.23
+          },
+          "17": {
+            "pass@1": 0.14675,
+            "pass@2": 0.16014972933070862,
+            "pass@4": 0.16990317519685033,
+            "pass@8": 0.17901342366976633,
+            "pass@16": 0.1890970246223302,
+            "pass@32": 0.2009162762964177,
+            "pass@64": 0.21379618773979528,
+            "pass@128": 0.227
+          },
+          "18": {
+            "pass@1": 0.123859375,
+            "pass@2": 0.13845374015748027,
+            "pass@4": 0.15128099362579675,
+            "pass@8": 0.1636793233057228,
+            "pass@16": 0.17590328285323045,
+            "pass@32": 0.18769288846945548,
+            "pass@64": 0.19939121996448883,
+            "pass@128": 0.213
+          },
+          "19": {
+            "pass@1": 0.1425859375,
+            "pass@2": 0.1542038631889764,
+            "pass@4": 0.16308084739407575,
+            "pass@8": 0.17024050661671622,
+            "pass@16": 0.17678480815036582,
+            "pass@32": 0.18411748530607766,
+            "pass@64": 0.19279372965274186,
+            "pass@128": 0.203
+          },
+          "20": {
+            "pass@1": 0.12134375,
+            "pass@2": 0.13410039370078738,
+            "pass@4": 0.1453615757405324,
+            "pass@8": 0.1559681872763174,
+            "pass@16": 0.1660694483193041,
+            "pass@32": 0.17588726848327202,
+            "pass@64": 0.1855413942815293,
+            "pass@128": 0.196
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      }
+    }
+  },
+  "total": {
+    "answer_accuracy": 0.24445970394736843,
+    "avg_loss": 0.0,
+    "count": 2432000,
+    "validation_metrics": {
+      "id": {},
+      "ood": {}
+    },
+    "pass_at_k": {
+      "pass@1": 0.24445970394736843,
+      "pass@2": 0.2560060803460411,
+      "pass@4": 0.2656400593067308,
+      "pass@8": 0.2747226211406185,
+      "pass@16": 0.2838467221654667,
+      "pass@32": 0.2934078672886908,
+      "pass@64": 0.30357656007471606,
+      "pass@128": 0.31463157894736843
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.179671875,
+        "pass@2": 0.1866203248031496,
+        "pass@4": 0.19349498397075368,
+        "pass@8": 0.20171124100488724,
+        "pass@16": 0.21184839720232201,
+        "pass@32": 0.22407444095023307,
+        "pass@64": 0.23828823396176474,
+        "pass@128": 0.255
+      },
+      "2": {
+        "pass@1": 0.9675390625,
+        "pass@2": 0.9754135088582678,
+        "pass@4": 0.9807076276715406,
+        "pass@8": 0.9842055326995033,
+        "pass@16": 0.9866531838082899,
+        "pass@32": 0.9888710784240634,
+        "pass@64": 0.991123358518923,
+        "pass@128": 0.993
+      },
+      "3": {
+        "pass@1": 0.7970390625,
+        "pass@2": 0.8376731053149615,
+        "pass@4": 0.8661379510686164,
+        "pass@8": 0.8884730983835463,
+        "pass@16": 0.9065882701957276,
+        "pass@32": 0.9221857999677552,
+        "pass@64": 0.9361523489727314,
+        "pass@128": 0.949
+      },
+      "4": {
+        "pass@1": 0.2107890625,
+        "pass@2": 0.21529441437007876,
+        "pass@4": 0.21916330239970003,
+        "pass@8": 0.22346821216105478,
+        "pass@16": 0.22872197474530756,
+        "pass@32": 0.23562179093658991,
+        "pass@64": 0.24510618631947784,
+        "pass@128": 0.258
+      },
+      "5": {
+        "pass@1": 0.2116171875,
+        "pass@2": 0.21723412893700783,
+        "pass@4": 0.22284044347581547,
+        "pass@8": 0.22918635588080552,
+        "pass@16": 0.23692081774417614,
+        "pass@32": 0.24659684355460448,
+        "pass@64": 0.25860912185373264,
+        "pass@128": 0.272
+      },
+      "6": {
+        "pass@1": 0.2580546875,
+        "pass@2": 0.262111343503937,
+        "pass@4": 0.26583126621672293,
+        "pass@8": 0.269561781409488,
+        "pass@16": 0.2735911486266416,
+        "pass@32": 0.2786441597934989,
+        "pass@64": 0.2852150182516317,
+        "pass@128": 0.293
+      },
+      "7": {
+        "pass@1": 0.195703125,
+        "pass@2": 0.2011721210629921,
+        "pass@4": 0.20649363198350204,
+        "pass@8": 0.212816902115117,
+        "pass@16": 0.22024960690112913,
+        "pass@32": 0.2283513294970211,
+        "pass@64": 0.23693507099206224,
+        "pass@128": 0.247
+      },
+      "8": {
+        "pass@1": 0.18925,
+        "pass@2": 0.19675135334645666,
+        "pass@4": 0.20417071484814395,
+        "pass@8": 0.21275267862195865,
+        "pass@16": 0.22278762964396398,
+        "pass@32": 0.23415911788441254,
+        "pass@64": 0.2463119989893093,
+        "pass@128": 0.258
+      },
+      "9": {
+        "pass@1": 0.175953125,
+        "pass@2": 0.18321788877952747,
+        "pass@4": 0.1904768334270717,
+        "pass@8": 0.19806054813384352,
+        "pass@16": 0.20626457792569045,
+        "pass@32": 0.21481318234518765,
+        "pass@64": 0.22277015534238243,
+        "pass@128": 0.231
+      },
+      "11": {
+        "pass@1": 0.1763515625,
+        "pass@2": 0.1854534940944882,
+        "pass@4": 0.1933735462129733,
+        "pass@8": 0.20147957136226124,
+        "pass@16": 0.21037463664286407,
+        "pass@32": 0.22037157099835816,
+        "pass@64": 0.23143147890828927,
+        "pass@128": 0.243
+      },
+      "12": {
+        "pass@1": 0.1750078125,
+        "pass@2": 0.18632972440944887,
+        "pass@4": 0.19674250149981243,
+        "pass@8": 0.2075677772984937,
+        "pass@16": 0.21899942949146833,
+        "pass@32": 0.23008881738333872,
+        "pass@64": 0.24004304392698875,
+        "pass@128": 0.249
+      },
+      "13": {
+        "pass@1": 0.14003125,
+        "pass@2": 0.15481865157480312,
+        "pass@4": 0.16692630361829766,
+        "pass@8": 0.17772499892830299,
+        "pass@16": 0.1882665689521189,
+        "pass@32": 0.19883793436883665,
+        "pass@64": 0.20967385171136876,
+        "pass@128": 0.221
+      },
+      "14": {
+        "pass@1": 0.1436328125,
+        "pass@2": 0.15869389763779526,
+        "pass@4": 0.17226924925009368,
+        "pass@8": 0.1845543174527869,
+        "pass@16": 0.19568671047542624,
+        "pass@32": 0.2065402438692653,
+        "pass@64": 0.21755881523205667,
+        "pass@128": 0.229
+      },
+      "15": {
+        "pass@1": 0.1338828125,
+        "pass@2": 0.14767212106299207,
+        "pass@4": 0.1592121984439444,
+        "pass@8": 0.1693664968394039,
+        "pass@16": 0.1790848548813016,
+        "pass@32": 0.1888553870136373,
+        "pass@64": 0.19929553877537814,
+        "pass@128": 0.211
+      },
+      "16": {
+        "pass@1": 0.155671875,
+        "pass@2": 0.16875172244094488,
+        "pass@4": 0.17969398078365192,
+        "pass@8": 0.1898988485117523,
+        "pass@16": 0.1991953499621563,
+        "pass@32": 0.20812386294309929,
+        "pass@64": 0.21791788802494516,
+        "pass@128": 0.23
+      },
+      "17": {
+        "pass@1": 0.14675,
+        "pass@2": 0.16014972933070862,
+        "pass@4": 0.16990317519685033,
+        "pass@8": 0.17901342366976633,
+        "pass@16": 0.1890970246223302,
+        "pass@32": 0.2009162762964177,
+        "pass@64": 0.21379618773979528,
+        "pass@128": 0.227
+      },
+      "18": {
+        "pass@1": 0.123859375,
+        "pass@2": 0.13845374015748027,
+        "pass@4": 0.15128099362579675,
+        "pass@8": 0.1636793233057228,
+        "pass@16": 0.17590328285323045,
+        "pass@32": 0.18769288846945548,
+        "pass@64": 0.19939121996448883,
+        "pass@128": 0.213
+      },
+      "19": {
+        "pass@1": 0.1425859375,
+        "pass@2": 0.1542038631889764,
+        "pass@4": 0.16308084739407575,
+        "pass@8": 0.17024050661671622,
+        "pass@16": 0.17678480815036582,
+        "pass@32": 0.18411748530607766,
+        "pass@64": 0.19279372965274186,
+        "pass@128": 0.203
+      },
+      "20": {
+        "pass@1": 0.12134375,
+        "pass@2": 0.13410039370078738,
+        "pass@4": 0.1453615757405324,
+        "pass@8": 0.1559681872763174,
+        "pass@16": 0.1660694483193041,
+        "pass@32": 0.17588726848327202,
+        "pass@64": 0.1855413942815293,
+        "pass@128": 0.196
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.179671875,
+      "2": 0.9675390625,
+      "3": 0.7970390625,
+      "4": 0.2107890625,
+      "5": 0.2116171875,
+      "6": 0.2580546875,
+      "7": 0.195703125,
+      "8": 0.18925,
+      "9": 0.175953125,
+      "11": 0.1763515625,
+      "12": 0.1750078125,
+      "13": 0.14003125,
+      "14": 0.1436328125,
+      "15": 0.1338828125,
+      "16": 0.155671875,
+      "17": 0.14675,
+      "18": 0.123859375,
+      "19": 0.1425859375,
+      "20": 0.12134375
+    },
+    "per_op_avg_response_len": {
+      "10": 167.241765625,
+      "2": 111.679328125,
+      "3": 128.0244453125,
+      "4": 125.2879921875,
+      "5": 135.821828125,
+      "6": 152.88628125,
+      "7": 142.7650703125,
+      "8": 163.613359375,
+      "9": 168.1250078125,
+      "11": 175.3768046875,
+      "12": 176.2021953125,
+      "13": 176.847875,
+      "14": 195.620578125,
+      "15": 203.4362421875,
+      "16": 206.30290625,
+      "17": 220.5351796875,
+      "18": 222.7138984375,
+      "19": 220.836265625,
+      "20": 227.097609375
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 167.241765625,
+      "2": 111.679328125,
+      "3": 128.0244453125,
+      "4": 125.2879921875,
+      "5": 135.821828125,
+      "6": 152.88628125,
+      "7": 142.7650703125,
+      "8": 163.613359375,
+      "9": 168.1250078125,
+      "11": 175.3768046875,
+      "12": 176.2021953125,
+      "13": 176.847875,
+      "14": 195.620578125,
+      "15": 203.4362421875,
+      "16": 206.30290625,
+      "17": 220.5351796875,
+      "18": 222.7138984375,
+      "19": 220.836265625,
+      "20": 227.097609375
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      },
+      "teachers_in_school": {
+        "count": 2432000,
+        "correct": 594526,
+        "answer_accuracy": 0.24445970394736843,
+        "avg_response_len": 174.7586648848684,
+        "resp_tokens_sum": 425013073,
+        "pass_at_k": {
+          "pass@1": 0.24445970394736843,
+          "pass@2": 0.2560060803460411,
+          "pass@4": 0.2656400593067308,
+          "pass@8": 0.2747226211406185,
+          "pass@16": 0.2838467221654667,
+          "pass@32": 0.2934078672886908,
+          "pass@64": 0.30357656007471606,
+          "pass@128": 0.31463157894736843
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.179671875,
+            "pass@2": 0.1866203248031496,
+            "pass@4": 0.19349498397075368,
+            "pass@8": 0.20171124100488724,
+            "pass@16": 0.21184839720232201,
+            "pass@32": 0.22407444095023307,
+            "pass@64": 0.23828823396176474,
+            "pass@128": 0.255
+          },
+          "2": {
+            "pass@1": 0.9675390625,
+            "pass@2": 0.9754135088582678,
+            "pass@4": 0.9807076276715406,
+            "pass@8": 0.9842055326995033,
+            "pass@16": 0.9866531838082899,
+            "pass@32": 0.9888710784240634,
+            "pass@64": 0.991123358518923,
+            "pass@128": 0.993
+          },
+          "3": {
+            "pass@1": 0.7970390625,
+            "pass@2": 0.8376731053149615,
+            "pass@4": 0.8661379510686164,
+            "pass@8": 0.8884730983835463,
+            "pass@16": 0.9065882701957276,
+            "pass@32": 0.9221857999677552,
+            "pass@64": 0.9361523489727314,
+            "pass@128": 0.949
+          },
+          "4": {
+            "pass@1": 0.2107890625,
+            "pass@2": 0.21529441437007876,
+            "pass@4": 0.21916330239970003,
+            "pass@8": 0.22346821216105478,
+            "pass@16": 0.22872197474530756,
+            "pass@32": 0.23562179093658991,
+            "pass@64": 0.24510618631947784,
+            "pass@128": 0.258
+          },
+          "5": {
+            "pass@1": 0.2116171875,
+            "pass@2": 0.21723412893700783,
+            "pass@4": 0.22284044347581547,
+            "pass@8": 0.22918635588080552,
+            "pass@16": 0.23692081774417614,
+            "pass@32": 0.24659684355460448,
+            "pass@64": 0.25860912185373264,
+            "pass@128": 0.272
+          },
+          "6": {
+            "pass@1": 0.2580546875,
+            "pass@2": 0.262111343503937,
+            "pass@4": 0.26583126621672293,
+            "pass@8": 0.269561781409488,
+            "pass@16": 0.2735911486266416,
+            "pass@32": 0.2786441597934989,
+            "pass@64": 0.2852150182516317,
+            "pass@128": 0.293
+          },
+          "7": {
+            "pass@1": 0.195703125,
+            "pass@2": 0.2011721210629921,
+            "pass@4": 0.20649363198350204,
+            "pass@8": 0.212816902115117,
+            "pass@16": 0.22024960690112913,
+            "pass@32": 0.2283513294970211,
+            "pass@64": 0.23693507099206224,
+            "pass@128": 0.247
+          },
+          "8": {
+            "pass@1": 0.18925,
+            "pass@2": 0.19675135334645666,
+            "pass@4": 0.20417071484814395,
+            "pass@8": 0.21275267862195865,
+            "pass@16": 0.22278762964396398,
+            "pass@32": 0.23415911788441254,
+            "pass@64": 0.2463119989893093,
+            "pass@128": 0.258
+          },
+          "9": {
+            "pass@1": 0.175953125,
+            "pass@2": 0.18321788877952747,
+            "pass@4": 0.1904768334270717,
+            "pass@8": 0.19806054813384352,
+            "pass@16": 0.20626457792569045,
+            "pass@32": 0.21481318234518765,
+            "pass@64": 0.22277015534238243,
+            "pass@128": 0.231
+          },
+          "11": {
+            "pass@1": 0.1763515625,
+            "pass@2": 0.1854534940944882,
+            "pass@4": 0.1933735462129733,
+            "pass@8": 0.20147957136226124,
+            "pass@16": 0.21037463664286407,
+            "pass@32": 0.22037157099835816,
+            "pass@64": 0.23143147890828927,
+            "pass@128": 0.243
+          },
+          "12": {
+            "pass@1": 0.1750078125,
+            "pass@2": 0.18632972440944887,
+            "pass@4": 0.19674250149981243,
+            "pass@8": 0.2075677772984937,
+            "pass@16": 0.21899942949146833,
+            "pass@32": 0.23008881738333872,
+            "pass@64": 0.24004304392698875,
+            "pass@128": 0.249
+          },
+          "13": {
+            "pass@1": 0.14003125,
+            "pass@2": 0.15481865157480312,
+            "pass@4": 0.16692630361829766,
+            "pass@8": 0.17772499892830299,
+            "pass@16": 0.1882665689521189,
+            "pass@32": 0.19883793436883665,
+            "pass@64": 0.20967385171136876,
+            "pass@128": 0.221
+          },
+          "14": {
+            "pass@1": 0.1436328125,
+            "pass@2": 0.15869389763779526,
+            "pass@4": 0.17226924925009368,
+            "pass@8": 0.1845543174527869,
+            "pass@16": 0.19568671047542624,
+            "pass@32": 0.2065402438692653,
+            "pass@64": 0.21755881523205667,
+            "pass@128": 0.229
+          },
+          "15": {
+            "pass@1": 0.1338828125,
+            "pass@2": 0.14767212106299207,
+            "pass@4": 0.1592121984439444,
+            "pass@8": 0.1693664968394039,
+            "pass@16": 0.1790848548813016,
+            "pass@32": 0.1888553870136373,
+            "pass@64": 0.19929553877537814,
+            "pass@128": 0.211
+          },
+          "16": {
+            "pass@1": 0.155671875,
+            "pass@2": 0.16875172244094488,
+            "pass@4": 0.17969398078365192,
+            "pass@8": 0.1898988485117523,
+            "pass@16": 0.1991953499621563,
+            "pass@32": 0.20812386294309929,
+            "pass@64": 0.21791788802494516,
+            "pass@128": 0.23
+          },
+          "17": {
+            "pass@1": 0.14675,
+            "pass@2": 0.16014972933070862,
+            "pass@4": 0.16990317519685033,
+            "pass@8": 0.17901342366976633,
+            "pass@16": 0.1890970246223302,
+            "pass@32": 0.2009162762964177,
+            "pass@64": 0.21379618773979528,
+            "pass@128": 0.227
+          },
+          "18": {
+            "pass@1": 0.123859375,
+            "pass@2": 0.13845374015748027,
+            "pass@4": 0.15128099362579675,
+            "pass@8": 0.1636793233057228,
+            "pass@16": 0.17590328285323045,
+            "pass@32": 0.18769288846945548,
+            "pass@64": 0.19939121996448883,
+            "pass@128": 0.213
+          },
+          "19": {
+            "pass@1": 0.1425859375,
+            "pass@2": 0.1542038631889764,
+            "pass@4": 0.16308084739407575,
+            "pass@8": 0.17024050661671622,
+            "pass@16": 0.17678480815036582,
+            "pass@32": 0.18411748530607766,
+            "pass@64": 0.19279372965274186,
+            "pass@128": 0.203
+          },
+          "20": {
+            "pass@1": 0.12134375,
+            "pass@2": 0.13410039370078738,
+            "pass@4": 0.1453615757405324,
+            "pass@8": 0.1559681872763174,
+            "pass@16": 0.1660694483193041,
+            "pass@32": 0.17588726848327202,
+            "pass@64": 0.1855413942815293,
+            "pass@128": 0.196
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      }
+    }
+  }
+}

Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/hf_model_ood_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6ef5b332824c945db1d9c25f006be194f9d3c2058bf85b03074d3b4d412bb39
+size 2223008342

Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/summary.csv

@@ -0,0 +1,2 @@
+checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
+hf_model,0.35395746527777777,0.14591171875,0.24445970394736843,0.0,0.0,0.0,143.9383420138889,202.49695546875,1152000,1280000,2432000

Qwen3-100M-RL/context/0.999zoo_op2-20+0.001teacher_op2/summary.json

@@ -0,0 +1,245 @@
+[
+  {
+    "checkpoint": "hf_model",
+    "id_acc": 0.35395746527777777,
+    "ood_acc": 0.14591171875,
+    "total_acc": 0.24445970394736843,
+    "id_avg_loss": 0.0,
+    "ood_avg_loss": 0.0,
+    "total_avg_loss": 0.0,
+    "id_avg_resp_len": 143.9383420138889,
+    "ood_avg_resp_len": 202.49695546875,
+    "count_id": 1152000,
+    "count_ood": 1280000,
+    "count_total": 2432000,
+    "template_metrics": {
+      "crazy_zootopia": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      },
+      "teachers_in_school": {
+        "count": 2432000,
+        "correct": 594526,
+        "answer_accuracy": 0.24445970394736843,
+        "avg_response_len": 174.7586648848684,
+        "resp_tokens_sum": 425013073,
+        "pass_at_k": {
+          "pass@1": 0.24445970394736843,
+          "pass@2": 0.2560060803460411,
+          "pass@4": 0.2656400593067308,
+          "pass@8": 0.2747226211406185,
+          "pass@16": 0.2838467221654667,
+          "pass@32": 0.2934078672886908,
+          "pass@64": 0.30357656007471606,
+          "pass@128": 0.31463157894736843
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.179671875,
+            "pass@2": 0.1866203248031496,
+            "pass@4": 0.19349498397075368,
+            "pass@8": 0.20171124100488724,
+            "pass@16": 0.21184839720232201,
+            "pass@32": 0.22407444095023307,
+            "pass@64": 0.23828823396176474,
+            "pass@128": 0.255
+          },
+          "2": {
+            "pass@1": 0.9675390625,
+            "pass@2": 0.9754135088582678,
+            "pass@4": 0.9807076276715406,
+            "pass@8": 0.9842055326995033,
+            "pass@16": 0.9866531838082899,
+            "pass@32": 0.9888710784240634,
+            "pass@64": 0.991123358518923,
+            "pass@128": 0.993
+          },
+          "3": {
+            "pass@1": 0.7970390625,
+            "pass@2": 0.8376731053149615,
+            "pass@4": 0.8661379510686164,
+            "pass@8": 0.8884730983835463,
+            "pass@16": 0.9065882701957276,
+            "pass@32": 0.9221857999677552,
+            "pass@64": 0.9361523489727314,
+            "pass@128": 0.949
+          },
+          "4": {
+            "pass@1": 0.2107890625,
+            "pass@2": 0.21529441437007876,
+            "pass@4": 0.21916330239970003,
+            "pass@8": 0.22346821216105478,
+            "pass@16": 0.22872197474530756,
+            "pass@32": 0.23562179093658991,
+            "pass@64": 0.24510618631947784,
+            "pass@128": 0.258
+          },
+          "5": {
+            "pass@1": 0.2116171875,
+            "pass@2": 0.21723412893700783,
+            "pass@4": 0.22284044347581547,
+            "pass@8": 0.22918635588080552,
+            "pass@16": 0.23692081774417614,
+            "pass@32": 0.24659684355460448,
+            "pass@64": 0.25860912185373264,
+            "pass@128": 0.272
+          },
+          "6": {
+            "pass@1": 0.2580546875,
+            "pass@2": 0.262111343503937,
+            "pass@4": 0.26583126621672293,
+            "pass@8": 0.269561781409488,
+            "pass@16": 0.2735911486266416,
+            "pass@32": 0.2786441597934989,
+            "pass@64": 0.2852150182516317,
+            "pass@128": 0.293
+          },
+          "7": {
+            "pass@1": 0.195703125,
+            "pass@2": 0.2011721210629921,
+            "pass@4": 0.20649363198350204,
+            "pass@8": 0.212816902115117,
+            "pass@16": 0.22024960690112913,
+            "pass@32": 0.2283513294970211,
+            "pass@64": 0.23693507099206224,
+            "pass@128": 0.247
+          },
+          "8": {
+            "pass@1": 0.18925,
+            "pass@2": 0.19675135334645666,
+            "pass@4": 0.20417071484814395,
+            "pass@8": 0.21275267862195865,
+            "pass@16": 0.22278762964396398,
+            "pass@32": 0.23415911788441254,
+            "pass@64": 0.2463119989893093,
+            "pass@128": 0.258
+          },
+          "9": {
+            "pass@1": 0.175953125,
+            "pass@2": 0.18321788877952747,
+            "pass@4": 0.1904768334270717,
+            "pass@8": 0.19806054813384352,
+            "pass@16": 0.20626457792569045,
+            "pass@32": 0.21481318234518765,
+            "pass@64": 0.22277015534238243,
+            "pass@128": 0.231
+          },
+          "11": {
+            "pass@1": 0.1763515625,
+            "pass@2": 0.1854534940944882,
+            "pass@4": 0.1933735462129733,
+            "pass@8": 0.20147957136226124,
+            "pass@16": 0.21037463664286407,
+            "pass@32": 0.22037157099835816,
+            "pass@64": 0.23143147890828927,
+            "pass@128": 0.243
+          },
+          "12": {
+            "pass@1": 0.1750078125,
+            "pass@2": 0.18632972440944887,
+            "pass@4": 0.19674250149981243,
+            "pass@8": 0.2075677772984937,
+            "pass@16": 0.21899942949146833,
+            "pass@32": 0.23008881738333872,
+            "pass@64": 0.24004304392698875,
+            "pass@128": 0.249
+          },
+          "13": {
+            "pass@1": 0.14003125,
+            "pass@2": 0.15481865157480312,
+            "pass@4": 0.16692630361829766,
+            "pass@8": 0.17772499892830299,
+            "pass@16": 0.1882665689521189,
+            "pass@32": 0.19883793436883665,
+            "pass@64": 0.20967385171136876,
+            "pass@128": 0.221
+          },
+          "14": {
+            "pass@1": 0.1436328125,
+            "pass@2": 0.15869389763779526,
+            "pass@4": 0.17226924925009368,
+            "pass@8": 0.1845543174527869,
+            "pass@16": 0.19568671047542624,
+            "pass@32": 0.2065402438692653,
+            "pass@64": 0.21755881523205667,
+            "pass@128": 0.229
+          },
+          "15": {
+            "pass@1": 0.1338828125,
+            "pass@2": 0.14767212106299207,
+            "pass@4": 0.1592121984439444,
+            "pass@8": 0.1693664968394039,
+            "pass@16": 0.1790848548813016,
+            "pass@32": 0.1888553870136373,
+            "pass@64": 0.19929553877537814,
+            "pass@128": 0.211
+          },
+          "16": {
+            "pass@1": 0.155671875,
+            "pass@2": 0.16875172244094488,
+            "pass@4": 0.17969398078365192,
+            "pass@8": 0.1898988485117523,
+            "pass@16": 0.1991953499621563,
+            "pass@32": 0.20812386294309929,
+            "pass@64": 0.21791788802494516,
+            "pass@128": 0.23
+          },
+          "17": {
+            "pass@1": 0.14675,
+            "pass@2": 0.16014972933070862,
+            "pass@4": 0.16990317519685033,
+            "pass@8": 0.17901342366976633,
+            "pass@16": 0.1890970246223302,
+            "pass@32": 0.2009162762964177,
+            "pass@64": 0.21379618773979528,
+            "pass@128": 0.227
+          },
+          "18": {
+            "pass@1": 0.123859375,
+            "pass@2": 0.13845374015748027,
+            "pass@4": 0.15128099362579675,
+            "pass@8": 0.1636793233057228,
+            "pass@16": 0.17590328285323045,
+            "pass@32": 0.18769288846945548,
+            "pass@64": 0.19939121996448883,
+            "pass@128": 0.213
+          },
+          "19": {
+            "pass@1": 0.1425859375,
+            "pass@2": 0.1542038631889764,
+            "pass@4": 0.16308084739407575,
+            "pass@8": 0.17024050661671622,
+            "pass@16": 0.17678480815036582,
+            "pass@32": 0.18411748530607766,
+            "pass@64": 0.19279372965274186,
+            "pass@128": 0.203
+          },
+          "20": {
+            "pass@1": 0.12134375,
+            "pass@2": 0.13410039370078738,
+            "pass@4": 0.1453615757405324,
+            "pass@8": 0.1559681872763174,
+            "pass@16": 0.1660694483193041,
+            "pass@32": 0.17588726848327202,
+            "pass@64": 0.1855413942815293,
+            "pass@128": 0.196
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      }
+    }
+  }
+]

Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "results/context/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2", "results_dir": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs", "per_file": [{"file": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 9541, "extra_nodes": 1152000, "missing_nodes": 1152000, "perfect_with_extra": 0, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "extra_nodes": 1152000, "missing_nodes": 1152000, "answer_mismatch": 9541}}, "results_path": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 121879, "extra_nodes": 1279999, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "extra_nodes": 1279999, "missing_nodes": 1280000, "answer_mismatch": 121879}}, "results_path": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 0, "dependency_mismatch": 0, "answer_mismatch": 131420, "extra_nodes": 2431999, "missing_nodes": 2432000, "perfect_with_extra": 0, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "extra_nodes": 2431999, "missing_nodes": 2432000, "answer_mismatch": 131420}}}}

Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/graphs/results.json

@@ -0,0 +1,910 @@
+{
+  "files": {
+    "hf_model_id_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "3": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "4": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "5": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "6": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "7": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "8": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "9": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "10": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@2": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@4": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@8": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@16": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@32": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@64": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@128": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "2": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "3": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "4": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "5": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "6": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "7": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "8": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "9": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "10": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@2": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@4": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@8": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@16": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@32": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@64": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@128": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.0,
+          "pass@2": 0.0,
+          "pass@4": 0.0,
+          "pass@8": 0.0,
+          "pass@16": 0.0,
+          "pass@32": 0.0,
+          "pass@64": 0.0,
+          "pass@128": 0.0
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "teachers_in_school": {
+            "per_op": {
+              "2": 1000,
+              "3": 1000,
+              "4": 1000,
+              "5": 1000,
+              "6": 1000,
+              "7": 1000,
+              "8": 1000,
+              "9": 1000,
+              "10": 1000
+            },
+            "total": 9000
+          }
+        },
+        "total": {
+          "per_op": {
+            "2": 1000,
+            "3": 1000,
+            "4": 1000,
+            "5": 1000,
+            "6": 1000,
+            "7": 1000,
+            "8": 1000,
+            "9": 1000,
+            "10": 1000
+          },
+          "total": 9000
+        }
+      }
+    },
+    "hf_model_ood_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "12": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "13": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "14": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "15": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "16": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "17": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "18": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "19": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "20": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@2": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@4": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@8": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@16": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@32": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@64": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@128": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "12": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "13": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "14": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "15": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "16": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "17": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "18": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "19": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "20": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@2": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@4": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@8": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@16": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@32": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@64": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@128": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.0,
+          "pass@2": 0.0,
+          "pass@4": 0.0,
+          "pass@8": 0.0,
+          "pass@16": 0.0,
+          "pass@32": 0.0,
+          "pass@64": 0.0,
+          "pass@128": 0.0
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "teachers_in_school": {
+            "per_op": {
+              "11": 1000,
+              "12": 1000,
+              "13": 1000,
+              "14": 1000,
+              "15": 1000,
+              "16": 1000,
+              "17": 1000,
+              "18": 1000,
+              "19": 1000,
+              "20": 1000
+            },
+            "total": 10000
+          }
+        },
+        "total": {
+          "per_op": {
+            "11": 1000,
+            "12": 1000,
+            "13": 1000,
+            "14": 1000,
+            "15": 1000,
+            "16": 1000,
+            "17": 1000,
+            "18": 1000,
+            "19": 1000,
+            "20": 1000
+          },
+          "total": 10000
+        }
+      }
+    }
+  }
+}

Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/hf_model_ood_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:968c7292d5ffa390a250d1268de665f37a960ee75282d094d9edc38122cbc9b8
+size 2616730794

Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/summary.csv

@@ -0,0 +1,2 @@
+checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
+hf_model,0.9917178819444444,0.90478203125,0.9459621710526316,0.0,0.0,0.0,180.0242638888889,346.5120359375,1152000,1280000,2432000

Qwen3-100M-RL/context/0.99zoo_op2-20+0.01teacher_op2/summary.json

@@ -0,0 +1,245 @@
+[
+  {
+    "checkpoint": "hf_model",
+    "id_acc": 0.9917178819444444,
+    "ood_acc": 0.90478203125,
+    "total_acc": 0.9459621710526316,
+    "id_avg_loss": 0.0,
+    "ood_avg_loss": 0.0,
+    "total_avg_loss": 0.0,
+    "id_avg_resp_len": 180.0242638888889,
+    "ood_avg_resp_len": 346.5120359375,
+    "count_id": 1152000,
+    "count_ood": 1280000,
+    "count_total": 2432000,
+    "template_metrics": {
+      "crazy_zootopia": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      },
+      "teachers_in_school": {
+        "count": 2432000,
+        "correct": 2300580,
+        "answer_accuracy": 0.9459621710526316,
+        "avg_response_len": 267.64940707236843,
+        "resp_tokens_sum": 650923358,
+        "pass_at_k": {
+          "pass@1": 0.9459621710526316,
+          "pass@2": 0.965499624430164,
+          "pass@4": 0.9754853252816995,
+          "pass@8": 0.9815746569203379,
+          "pass@16": 0.9856429402662464,
+          "pass@32": 0.9886553766902775,
+          "pass@64": 0.9910604803835107,
+          "pass@128": 0.9930526315789474
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9699453125,
+            "pass@2": 0.9797700541338586,
+            "pass@4": 0.9846088579865014,
+            "pass@8": 0.9883971297589192,
+            "pass@16": 0.991548693276063,
+            "pass@32": 0.9939062487742031,
+            "pass@64": 0.9956849154464417,
+            "pass@128": 0.997
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.99759375,
+            "pass@2": 0.9991465305118111,
+            "pass@4": 0.9998152384701912,
+            "pass@8": 0.9999862821828273,
+            "pass@16": 0.9999999227084759,
+            "pass@32": 0.9999999999993933,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.99834375,
+            "pass@2": 0.9994431594488189,
+            "pass@4": 0.9998681761342333,
+            "pass@8": 0.9999880140745279,
+            "pass@16": 0.9999999083912323,
+            "pass@32": 0.999999999998944,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9933515625,
+            "pass@2": 0.9955663139763777,
+            "pass@4": 0.9974079118860144,
+            "pass@8": 0.9986958248958838,
+            "pass@16": 0.9994353865491737,
+            "pass@32": 0.9998235513802302,
+            "pass@64": 0.9999861740539754,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.995625,
+            "pass@2": 0.99801156496063,
+            "pass@4": 0.9991621754780652,
+            "pass@8": 0.9997905041568963,
+            "pass@16": 0.9999826485802666,
+            "pass@32": 0.999999900717592,
+            "pass@64": 0.9999999999999175,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.991,
+            "pass@2": 0.9933371062992128,
+            "pass@4": 0.9947224583802023,
+            "pass@8": 0.9958695873510292,
+            "pass@16": 0.9966041585170369,
+            "pass@32": 0.9969280491355603,
+            "pass@64": 0.9969985543119962,
+            "pass@128": 0.997
+          },
+          "8": {
+            "pass@1": 0.9948046875,
+            "pass@2": 0.99785125492126,
+            "pass@4": 0.998795172384702,
+            "pass@8": 0.9994316174246192,
+            "pass@16": 0.9998579688324685,
+            "pass@32": 0.9999931255781963,
+            "pass@64": 0.9999999975930302,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.984796875,
+            "pass@2": 0.9912790354330716,
+            "pass@4": 0.9942763324896889,
+            "pass@8": 0.996222904016301,
+            "pass@16": 0.9974501804727003,
+            "pass@32": 0.9981885625139278,
+            "pass@64": 0.998723119840615,
+            "pass@128": 0.999
+          },
+          "11": {
+            "pass@1": 0.9512578125,
+            "pass@2": 0.9658013041338588,
+            "pass@4": 0.9731058290213717,
+            "pass@8": 0.9777503786832513,
+            "pass@16": 0.9812478554053372,
+            "pass@32": 0.9844936593333612,
+            "pass@64": 0.9875782128535292,
+            "pass@128": 0.99
+          },
+          "12": {
+            "pass@1": 0.9279609375,
+            "pass@2": 0.9452753444881897,
+            "pass@4": 0.9557707446569181,
+            "pass@8": 0.9629128336731604,
+            "pass@16": 0.9684001319561727,
+            "pass@32": 0.9734901999432878,
+            "pass@64": 0.9785937143414183,
+            "pass@128": 0.984
+          },
+          "13": {
+            "pass@1": 0.9148515625,
+            "pass@2": 0.9373233267716538,
+            "pass@4": 0.9503126651668536,
+            "pass@8": 0.9590150667585761,
+            "pass@16": 0.9658224358162538,
+            "pass@32": 0.9720045806193894,
+            "pass@64": 0.9776870791192475,
+            "pass@128": 0.983
+          },
+          "14": {
+            "pass@1": 0.9195390625,
+            "pass@2": 0.9461455462598434,
+            "pass@4": 0.9610489086989125,
+            "pass@8": 0.9705652801128191,
+            "pass@16": 0.976617495691548,
+            "pass@32": 0.9807539224865852,
+            "pass@64": 0.9838434201680727,
+            "pass@128": 0.986
+          },
+          "15": {
+            "pass@1": 0.9212421875,
+            "pass@2": 0.9496477608267725,
+            "pass@4": 0.9639471972253464,
+            "pass@8": 0.9731272495996897,
+            "pass@16": 0.9795135593649998,
+            "pass@32": 0.9843676266202894,
+            "pass@64": 0.9887092071015339,
+            "pass@128": 0.993
+          },
+          "16": {
+            "pass@1": 0.9186484375,
+            "pass@2": 0.9507684547244115,
+            "pass@4": 0.9659008573303339,
+            "pass@8": 0.974618462414321,
+            "pass@16": 0.9800429740397889,
+            "pass@32": 0.9841030640501913,
+            "pass@64": 0.9876616774077218,
+            "pass@128": 0.99
+          },
+          "17": {
+            "pass@1": 0.9137890625,
+            "pass@2": 0.9511459153543321,
+            "pass@4": 0.9683324656917881,
+            "pass@8": 0.9782519917436393,
+            "pass@16": 0.9845447476466959,
+            "pass@32": 0.9884888034847372,
+            "pass@64": 0.9909003607208057,
+            "pass@128": 0.993
+          },
+          "18": {
+            "pass@1": 0.8819921875,
+            "pass@2": 0.9288923474409468,
+            "pass@4": 0.9518024659730036,
+            "pass@8": 0.9643686704288694,
+            "pass@16": 0.9725036453284903,
+            "pass@32": 0.9785188430691222,
+            "pass@64": 0.9827390755534815,
+            "pass@128": 0.985
+          },
+          "19": {
+            "pass@1": 0.8668359375,
+            "pass@2": 0.9196662155511819,
+            "pass@4": 0.9460197001312345,
+            "pass@8": 0.9618559891779669,
+            "pass@16": 0.9724486989570319,
+            "pass@32": 0.9796740074314165,
+            "pass@64": 0.9850677359090562,
+            "pass@128": 0.99
+          },
+          "20": {
+            "pass@1": 0.831703125,
+            "pass@2": 0.8954216289370094,
+            "pass@4": 0.9293240232470946,
+            "pass@8": 0.9490706950329557,
+            "pass@16": 0.9611954535250025,
+            "pass@32": 0.9697180119787673,
+            "pass@64": 0.9759758828658656,
+            "pass@128": 0.981
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      }
+    }
+  }
+]

Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/0.9zoo_op2-20+0.1teacher_op2_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "results/context/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2", "results_dir": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs", "per_file": [{"file": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 2173, "dependency_mismatch": 933959, "answer_mismatch": 8758, "extra_nodes": 1151956, "missing_nodes": 1152000, "perfect_with_extra": 0, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "dependency_mismatch": 933959, "extra_nodes": 1151956, "missing_nodes": 1152000, "answer_mismatch": 8758, "value_mismatch": 2173}}, "results_path": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/hf_model_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 46724, "dependency_mismatch": 1062723, "answer_mismatch": 111672, "extra_nodes": 1279965, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "dependency_mismatch": 1062723, "answer_mismatch": 111672, "extra_nodes": 1279965, "missing_nodes": 1280000, "value_mismatch": 46724}}, "results_path": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/hf_model_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 48897, "dependency_mismatch": 1996682, "answer_mismatch": 120430, "extra_nodes": 2431921, "missing_nodes": 2432000, "perfect_with_extra": 0, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/0.9zoo_op2-20+0.1teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "dependency_mismatch": 1996682, "extra_nodes": 2431921, "missing_nodes": 2432000, "answer_mismatch": 120430, "value_mismatch": 48897}}}}

Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/graphs/results.json

@@ -0,0 +1,910 @@
+{
+  "files": {
+    "hf_model_id_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "3": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "4": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "5": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "6": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "7": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "8": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "9": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "10": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@2": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@4": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@8": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@16": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@32": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@64": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            },
+            "pass@128": {
+              "2": 0.0,
+              "3": 0.0,
+              "4": 0.0,
+              "5": 0.0,
+              "6": 0.0,
+              "7": 0.0,
+              "8": 0.0,
+              "9": 0.0,
+              "10": 0.0
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "2": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "3": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "4": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "5": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "6": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "7": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "8": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "9": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "10": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@2": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@4": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@8": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@16": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@32": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@64": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          },
+          "pass@128": {
+            "2": 0.0,
+            "3": 0.0,
+            "4": 0.0,
+            "5": 0.0,
+            "6": 0.0,
+            "7": 0.0,
+            "8": 0.0,
+            "9": 0.0,
+            "10": 0.0
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.0,
+          "pass@2": 0.0,
+          "pass@4": 0.0,
+          "pass@8": 0.0,
+          "pass@16": 0.0,
+          "pass@32": 0.0,
+          "pass@64": 0.0,
+          "pass@128": 0.0
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "teachers_in_school": {
+            "per_op": {
+              "2": 1000,
+              "3": 1000,
+              "4": 1000,
+              "5": 1000,
+              "6": 1000,
+              "7": 1000,
+              "8": 1000,
+              "9": 1000,
+              "10": 1000
+            },
+            "total": 9000
+          }
+        },
+        "total": {
+          "per_op": {
+            "2": 1000,
+            "3": 1000,
+            "4": 1000,
+            "5": 1000,
+            "6": 1000,
+            "7": 1000,
+            "8": 1000,
+            "9": 1000,
+            "10": 1000
+          },
+          "total": 9000
+        }
+      }
+    },
+    "hf_model_ood_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "12": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "13": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "14": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "15": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "16": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "17": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "18": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "19": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            },
+            "20": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.0,
+              "pass@16": 0.0,
+              "pass@32": 0.0,
+              "pass@64": 0.0,
+              "pass@128": 0.0
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@2": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@4": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@8": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@16": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@32": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@64": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            },
+            "pass@128": {
+              "11": 0.0,
+              "12": 0.0,
+              "13": 0.0,
+              "14": 0.0,
+              "15": 0.0,
+              "16": 0.0,
+              "17": 0.0,
+              "18": 0.0,
+              "19": 0.0,
+              "20": 0.0
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "12": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "13": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "14": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "15": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "16": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "17": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "18": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "19": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          },
+          "20": {
+            "pass@1": 0.0,
+            "pass@2": 0.0,
+            "pass@4": 0.0,
+            "pass@8": 0.0,
+            "pass@16": 0.0,
+            "pass@32": 0.0,
+            "pass@64": 0.0,
+            "pass@128": 0.0
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@2": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@4": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@8": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@16": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@32": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@64": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          },
+          "pass@128": {
+            "11": 0.0,
+            "12": 0.0,
+            "13": 0.0,
+            "14": 0.0,
+            "15": 0.0,
+            "16": 0.0,
+            "17": 0.0,
+            "18": 0.0,
+            "19": 0.0,
+            "20": 0.0
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.0,
+          "pass@2": 0.0,
+          "pass@4": 0.0,
+          "pass@8": 0.0,
+          "pass@16": 0.0,
+          "pass@32": 0.0,
+          "pass@64": 0.0,
+          "pass@128": 0.0
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "teachers_in_school": {
+            "per_op": {
+              "11": 1000,
+              "12": 1000,
+              "13": 1000,
+              "14": 1000,
+              "15": 1000,
+              "16": 1000,
+              "17": 1000,
+              "18": 1000,
+              "19": 1000,
+              "20": 1000
+            },
+            "total": 10000
+          }
+        },
+        "total": {
+          "per_op": {
+            "11": 1000,
+            "12": 1000,
+            "13": 1000,
+            "14": 1000,
+            "15": 1000,
+            "16": 1000,
+            "17": 1000,
+            "18": 1000,
+            "19": 1000,
+            "20": 1000
+          },
+          "total": 10000
+        }
+      }
+    }
+  }
+}

Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_id_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb59d98db815792924ef404ed5ac8271c437ee058e414b57938353dcb1adef5d
+size 1674664484

Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_metrics.json

@@ -0,0 +1,1077 @@
+{
+  "checkpoint": "hf_model",
+  "id": {
+    "answer_accuracy": 0.9923975694444445,
+    "avg_response_len": 176.4994105902778,
+    "avg_loss": 0.0,
+    "count": 1152000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.9923975694444445,
+      "pass@2": 0.9957820018591392,
+      "pass@4": 0.9971995788963887,
+      "pass@8": 0.9979632664900273,
+      "pass@16": 0.9984037349725341,
+      "pass@32": 0.9987061615186326,
+      "pass@64": 0.9988642449684446,
+      "pass@128": 0.9988888888888889
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.9722734375,
+        "pass@2": 0.982090920275591,
+        "pass@4": 0.9870395954255715,
+        "pass@8": 0.9898463912960546,
+        "pass@16": 0.9913574932933285,
+        "pass@32": 0.9924062358073099,
+        "pass@64": 0.9929435009525102,
+        "pass@128": 0.993
+      },
+      "2": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "3": {
+        "pass@1": 0.99696875,
+        "pass@2": 0.9987176427165356,
+        "pass@4": 0.999073522215973,
+        "pass@8": 0.9991772006158419,
+        "pass@16": 0.9993323490768988,
+        "pass@32": 0.9995814585676791,
+        "pass@64": 0.9998779527559055,
+        "pass@128": 1.0
+      },
+      "4": {
+        "pass@1": 0.998421875,
+        "pass@2": 0.999459153543307,
+        "pass@4": 0.9998440072178476,
+        "pass@8": 0.999979701409027,
+        "pass@16": 0.9999997111918157,
+        "pass@32": 0.9999999999851739,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "5": {
+        "pass@1": 0.9941015625,
+        "pass@2": 0.9965524114173228,
+        "pass@4": 0.9978783887326584,
+        "pass@8": 0.9987054335254999,
+        "pass@16": 0.9993742678247356,
+        "pass@32": 0.9998140401510567,
+        "pass@64": 0.9999861487158879,
+        "pass@128": 1.0
+      },
+      "6": {
+        "pass@1": 0.99475,
+        "pass@2": 0.9969265501968505,
+        "pass@4": 0.9980495649606299,
+        "pass@8": 0.9987365127580474,
+        "pass@16": 0.9989781223272445,
+        "pass@32": 0.9989999037739257,
+        "pass@64": 0.9989999999999666,
+        "pass@128": 0.999
+      },
+      "7": {
+        "pass@1": 0.9951015625,
+        "pass@2": 0.9976016240157483,
+        "pass@4": 0.9984762033183351,
+        "pass@8": 0.9988805544747335,
+        "pass@16": 0.9989943223963219,
+        "pass@32": 0.9989999945303923,
+        "pass@64": 0.999,
+        "pass@128": 0.999
+      },
+      "8": {
+        "pass@1": 0.992984375,
+        "pass@2": 0.9974980314960634,
+        "pass@4": 0.9988455806149232,
+        "pass@8": 0.9992881325051802,
+        "pass@16": 0.9995580380881292,
+        "pass@32": 0.999829071592898,
+        "pass@64": 0.9999861763011332,
+        "pass@128": 1.0
+      },
+      "9": {
+        "pass@1": 0.9869765625,
+        "pass@2": 0.9931916830708665,
+        "pass@4": 0.995589347581552,
+        "pass@8": 0.9970554718258637,
+        "pass@16": 0.9980393105543356,
+        "pass@32": 0.9987247492592555,
+        "pass@64": 0.998984425990599,
+        "pass@128": 0.999
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.9722734375,
+      "2": 1.0,
+      "3": 0.99696875,
+      "4": 0.998421875,
+      "5": 0.9941015625,
+      "6": 0.99475,
+      "7": 0.9951015625,
+      "8": 0.992984375,
+      "9": 0.9869765625
+    },
+    "per_op_avg_response_len": {
+      "10": 245.9643203125,
+      "2": 103.040046875,
+      "3": 121.6964375,
+      "4": 141.7835859375,
+      "5": 164.8098203125,
+      "6": 175.4171015625,
+      "7": 190.5570234375,
+      "8": 213.8619296875,
+      "9": 231.3644296875
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 245.9643203125,
+      "2": 103.040046875,
+      "3": 121.6964375,
+      "4": 141.7835859375,
+      "5": 164.8098203125,
+      "6": 175.4171015625,
+      "7": 190.5570234375,
+      "8": 213.8619296875,
+      "9": 231.3644296875
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      },
+      "teachers_in_school": {
+        "count": 1152000,
+        "correct": 1143242,
+        "answer_accuracy": 0.9923975694444445,
+        "avg_response_len": 176.4994105902778,
+        "resp_tokens_sum": 203327321,
+        "pass_at_k": {
+          "pass@1": 0.9923975694444445,
+          "pass@2": 0.9957820018591392,
+          "pass@4": 0.9971995788963887,
+          "pass@8": 0.9979632664900273,
+          "pass@16": 0.9984037349725341,
+          "pass@32": 0.9987061615186326,
+          "pass@64": 0.9988642449684446,
+          "pass@128": 0.9988888888888889
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9722734375,
+            "pass@2": 0.982090920275591,
+            "pass@4": 0.9870395954255715,
+            "pass@8": 0.9898463912960546,
+            "pass@16": 0.9913574932933285,
+            "pass@32": 0.9924062358073099,
+            "pass@64": 0.9929435009525102,
+            "pass@128": 0.993
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.99696875,
+            "pass@2": 0.9987176427165356,
+            "pass@4": 0.999073522215973,
+            "pass@8": 0.9991772006158419,
+            "pass@16": 0.9993323490768988,
+            "pass@32": 0.9995814585676791,
+            "pass@64": 0.9998779527559055,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.998421875,
+            "pass@2": 0.999459153543307,
+            "pass@4": 0.9998440072178476,
+            "pass@8": 0.999979701409027,
+            "pass@16": 0.9999997111918157,
+            "pass@32": 0.9999999999851739,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9941015625,
+            "pass@2": 0.9965524114173228,
+            "pass@4": 0.9978783887326584,
+            "pass@8": 0.9987054335254999,
+            "pass@16": 0.9993742678247356,
+            "pass@32": 0.9998140401510567,
+            "pass@64": 0.9999861487158879,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.99475,
+            "pass@2": 0.9969265501968505,
+            "pass@4": 0.9980495649606299,
+            "pass@8": 0.9987365127580474,
+            "pass@16": 0.9989781223272445,
+            "pass@32": 0.9989999037739257,
+            "pass@64": 0.9989999999999666,
+            "pass@128": 0.999
+          },
+          "7": {
+            "pass@1": 0.9951015625,
+            "pass@2": 0.9976016240157483,
+            "pass@4": 0.9984762033183351,
+            "pass@8": 0.9988805544747335,
+            "pass@16": 0.9989943223963219,
+            "pass@32": 0.9989999945303923,
+            "pass@64": 0.999,
+            "pass@128": 0.999
+          },
+          "8": {
+            "pass@1": 0.992984375,
+            "pass@2": 0.9974980314960634,
+            "pass@4": 0.9988455806149232,
+            "pass@8": 0.9992881325051802,
+            "pass@16": 0.9995580380881292,
+            "pass@32": 0.999829071592898,
+            "pass@64": 0.9999861763011332,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9869765625,
+            "pass@2": 0.9931916830708665,
+            "pass@4": 0.995589347581552,
+            "pass@8": 0.9970554718258637,
+            "pass@16": 0.9980393105543356,
+            "pass@32": 0.9987247492592555,
+            "pass@64": 0.998984425990599,
+            "pass@128": 0.999
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      }
+    }
+  },
+  "ood": {
+    "answer_accuracy": 0.91275625,
+    "avg_response_len": 339.53681640625,
+    "avg_loss": 0.0,
+    "count": 1280000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.91275625,
+      "pass@2": 0.9458462967519363,
+      "pass@4": 0.9624475184758139,
+      "pass@8": 0.9726641426761587,
+      "pass@16": 0.9796753897332539,
+      "pass@32": 0.9849077514114102,
+      "pass@64": 0.988984272764239,
+      "pass@128": 0.9919
+    },
+    "per_op_pass_at_k": {
+      "11": {
+        "pass@1": 0.9569375,
+        "pass@2": 0.9694979084645677,
+        "pass@4": 0.9755786563554552,
+        "pass@8": 0.9795771285286979,
+        "pass@16": 0.9828716597217099,
+        "pass@32": 0.9858994698672985,
+        "pass@64": 0.9886172988906358,
+        "pass@128": 0.991
+      },
+      "12": {
+        "pass@1": 0.9321875,
+        "pass@2": 0.9527299458661426,
+        "pass@4": 0.964783362579677,
+        "pass@8": 0.9725947365272651,
+        "pass@16": 0.9780288578767546,
+        "pass@32": 0.9822927738860049,
+        "pass@64": 0.9860262163530781,
+        "pass@128": 0.989
+      },
+      "13": {
+        "pass@1": 0.9148515625,
+        "pass@2": 0.9376562500000003,
+        "pass@4": 0.9520333217097862,
+        "pass@8": 0.9627994330286813,
+        "pass@16": 0.9713319331348897,
+        "pass@32": 0.9782062595316485,
+        "pass@64": 0.9837086690994824,
+        "pass@128": 0.988
+      },
+      "14": {
+        "pass@1": 0.92659375,
+        "pass@2": 0.9529224901574813,
+        "pass@4": 0.9665270294338203,
+        "pass@8": 0.9751033296649382,
+        "pass@16": 0.9814160863652701,
+        "pass@32": 0.9866717919871326,
+        "pass@64": 0.99098669283729,
+        "pass@128": 0.993
+      },
+      "15": {
+        "pass@1": 0.9321484375,
+        "pass@2": 0.9586374261811037,
+        "pass@4": 0.9716738764529432,
+        "pass@8": 0.980257322244576,
+        "pass@16": 0.9860876075856799,
+        "pass@32": 0.9905182347017089,
+        "pass@64": 0.9941053539635863,
+        "pass@128": 0.997
+      },
+      "16": {
+        "pass@1": 0.919421875,
+        "pass@2": 0.9510446604330718,
+        "pass@4": 0.9672112351893514,
+        "pass@8": 0.9771642191649528,
+        "pass@16": 0.9837104050806683,
+        "pass@32": 0.9884127699664014,
+        "pass@64": 0.9923083437843704,
+        "pass@128": 0.995
+      },
+      "17": {
+        "pass@1": 0.917234375,
+        "pass@2": 0.9551825787401591,
+        "pass@4": 0.9728983112110978,
+        "pass@8": 0.9831834738064922,
+        "pass@16": 0.989503100920689,
+        "pass@32": 0.9936977352294757,
+        "pass@64": 0.9968179233919849,
+        "pass@128": 0.999
+      },
+      "18": {
+        "pass@1": 0.8954921875,
+        "pass@2": 0.9377255167322847,
+        "pass@4": 0.9575529777840263,
+        "pass@8": 0.9690916457538964,
+        "pass@16": 0.9765775044869797,
+        "pass@32": 0.9819170635269723,
+        "pass@64": 0.9859361791907656,
+        "pass@128": 0.989
+      },
+      "19": {
+        "pass@1": 0.875109375,
+        "pass@2": 0.9267666092519707,
+        "pass@4": 0.9520276386389189,
+        "pass@8": 0.9663416476690758,
+        "pass@16": 0.975462333146504,
+        "pass@32": 0.981573341392933,
+        "pass@64": 0.9858043859763082,
+        "pass@128": 0.989
+      },
+      "20": {
+        "pass@1": 0.8575859375,
+        "pass@2": 0.9162995816929158,
+        "pass@4": 0.944188775403074,
+        "pass@8": 0.9605284903730037,
+        "pass@16": 0.9717644090133872,
+        "pass@32": 0.9798880740245233,
+        "pass@64": 0.9855316641548822,
+        "pass@128": 0.989
+      }
+    },
+    "per_op_accuracy": {
+      "11": 0.9569375,
+      "12": 0.9321875,
+      "13": 0.9148515625,
+      "14": 0.92659375,
+      "15": 0.9321484375,
+      "16": 0.919421875,
+      "17": 0.917234375,
+      "18": 0.8954921875,
+      "19": 0.875109375,
+      "20": 0.8575859375
+    },
+    "per_op_avg_response_len": {
+      "11": 266.557984375,
+      "12": 278.4357265625,
+      "13": 288.8397265625,
+      "14": 317.2391484375,
+      "15": 331.4378515625,
+      "16": 357.8768828125,
+      "17": 368.6404765625,
+      "18": 382.5142890625,
+      "19": 393.4576171875,
+      "20": 410.3684609375
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "11": 266.557984375,
+      "12": 278.4357265625,
+      "13": 288.8397265625,
+      "14": 317.2391484375,
+      "15": 331.4378515625,
+      "16": 357.8768828125,
+      "17": 368.6404765625,
+      "18": 382.5142890625,
+      "19": 393.4576171875,
+      "20": 410.3684609375
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      },
+      "teachers_in_school": {
+        "count": 1280000,
+        "correct": 1168328,
+        "answer_accuracy": 0.91275625,
+        "avg_response_len": 339.53681640625,
+        "resp_tokens_sum": 434607125,
+        "pass_at_k": {
+          "pass@1": 0.91275625,
+          "pass@2": 0.9458462967519363,
+          "pass@4": 0.9624475184758139,
+          "pass@8": 0.9726641426761587,
+          "pass@16": 0.9796753897332539,
+          "pass@32": 0.9849077514114102,
+          "pass@64": 0.988984272764239,
+          "pass@128": 0.9919
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9569375,
+            "pass@2": 0.9694979084645677,
+            "pass@4": 0.9755786563554552,
+            "pass@8": 0.9795771285286979,
+            "pass@16": 0.9828716597217099,
+            "pass@32": 0.9858994698672985,
+            "pass@64": 0.9886172988906358,
+            "pass@128": 0.991
+          },
+          "12": {
+            "pass@1": 0.9321875,
+            "pass@2": 0.9527299458661426,
+            "pass@4": 0.964783362579677,
+            "pass@8": 0.9725947365272651,
+            "pass@16": 0.9780288578767546,
+            "pass@32": 0.9822927738860049,
+            "pass@64": 0.9860262163530781,
+            "pass@128": 0.989
+          },
+          "13": {
+            "pass@1": 0.9148515625,
+            "pass@2": 0.9376562500000003,
+            "pass@4": 0.9520333217097862,
+            "pass@8": 0.9627994330286813,
+            "pass@16": 0.9713319331348897,
+            "pass@32": 0.9782062595316485,
+            "pass@64": 0.9837086690994824,
+            "pass@128": 0.988
+          },
+          "14": {
+            "pass@1": 0.92659375,
+            "pass@2": 0.9529224901574813,
+            "pass@4": 0.9665270294338203,
+            "pass@8": 0.9751033296649382,
+            "pass@16": 0.9814160863652701,
+            "pass@32": 0.9866717919871326,
+            "pass@64": 0.99098669283729,
+            "pass@128": 0.993
+          },
+          "15": {
+            "pass@1": 0.9321484375,
+            "pass@2": 0.9586374261811037,
+            "pass@4": 0.9716738764529432,
+            "pass@8": 0.980257322244576,
+            "pass@16": 0.9860876075856799,
+            "pass@32": 0.9905182347017089,
+            "pass@64": 0.9941053539635863,
+            "pass@128": 0.997
+          },
+          "16": {
+            "pass@1": 0.919421875,
+            "pass@2": 0.9510446604330718,
+            "pass@4": 0.9672112351893514,
+            "pass@8": 0.9771642191649528,
+            "pass@16": 0.9837104050806683,
+            "pass@32": 0.9884127699664014,
+            "pass@64": 0.9923083437843704,
+            "pass@128": 0.995
+          },
+          "17": {
+            "pass@1": 0.917234375,
+            "pass@2": 0.9551825787401591,
+            "pass@4": 0.9728983112110978,
+            "pass@8": 0.9831834738064922,
+            "pass@16": 0.989503100920689,
+            "pass@32": 0.9936977352294757,
+            "pass@64": 0.9968179233919849,
+            "pass@128": 0.999
+          },
+          "18": {
+            "pass@1": 0.8954921875,
+            "pass@2": 0.9377255167322847,
+            "pass@4": 0.9575529777840263,
+            "pass@8": 0.9690916457538964,
+            "pass@16": 0.9765775044869797,
+            "pass@32": 0.9819170635269723,
+            "pass@64": 0.9859361791907656,
+            "pass@128": 0.989
+          },
+          "19": {
+            "pass@1": 0.875109375,
+            "pass@2": 0.9267666092519707,
+            "pass@4": 0.9520276386389189,
+            "pass@8": 0.9663416476690758,
+            "pass@16": 0.975462333146504,
+            "pass@32": 0.981573341392933,
+            "pass@64": 0.9858043859763082,
+            "pass@128": 0.989
+          },
+          "20": {
+            "pass@1": 0.8575859375,
+            "pass@2": 0.9162995816929158,
+            "pass@4": 0.944188775403074,
+            "pass@8": 0.9605284903730037,
+            "pass@16": 0.9717644090133872,
+            "pass@32": 0.9798880740245233,
+            "pass@64": 0.9855316641548822,
+            "pass@128": 0.989
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      }
+    }
+  },
+  "total": {
+    "answer_accuracy": 0.9504810855263158,
+    "avg_loss": 0.0,
+    "count": 2432000,
+    "validation_metrics": {
+      "id": {},
+      "ood": {}
+    },
+    "pass_at_k": {
+      "pass@1": 0.9504810855263158,
+      "pass@2": 0.9695000518027258,
+      "pass@4": 0.9789090207802891,
+      "pass@8": 0.9846479381669422,
+      "pass@16": 0.988546711162386,
+      "pass@32": 0.9914438404095707,
+      "pass@64": 0.9936642595978096,
+      "pass@128": 0.9952105263157894
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.9722734375,
+        "pass@2": 0.982090920275591,
+        "pass@4": 0.9870395954255715,
+        "pass@8": 0.9898463912960546,
+        "pass@16": 0.9913574932933285,
+        "pass@32": 0.9924062358073099,
+        "pass@64": 0.9929435009525102,
+        "pass@128": 0.993
+      },
+      "2": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "3": {
+        "pass@1": 0.99696875,
+        "pass@2": 0.9987176427165356,
+        "pass@4": 0.999073522215973,
+        "pass@8": 0.9991772006158419,
+        "pass@16": 0.9993323490768988,
+        "pass@32": 0.9995814585676791,
+        "pass@64": 0.9998779527559055,
+        "pass@128": 1.0
+      },
+      "4": {
+        "pass@1": 0.998421875,
+        "pass@2": 0.999459153543307,
+        "pass@4": 0.9998440072178476,
+        "pass@8": 0.999979701409027,
+        "pass@16": 0.9999997111918157,
+        "pass@32": 0.9999999999851739,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "5": {
+        "pass@1": 0.9941015625,
+        "pass@2": 0.9965524114173228,
+        "pass@4": 0.9978783887326584,
+        "pass@8": 0.9987054335254999,
+        "pass@16": 0.9993742678247356,
+        "pass@32": 0.9998140401510567,
+        "pass@64": 0.9999861487158879,
+        "pass@128": 1.0
+      },
+      "6": {
+        "pass@1": 0.99475,
+        "pass@2": 0.9969265501968505,
+        "pass@4": 0.9980495649606299,
+        "pass@8": 0.9987365127580474,
+        "pass@16": 0.9989781223272445,
+        "pass@32": 0.9989999037739257,
+        "pass@64": 0.9989999999999666,
+        "pass@128": 0.999
+      },
+      "7": {
+        "pass@1": 0.9951015625,
+        "pass@2": 0.9976016240157483,
+        "pass@4": 0.9984762033183351,
+        "pass@8": 0.9988805544747335,
+        "pass@16": 0.9989943223963219,
+        "pass@32": 0.9989999945303923,
+        "pass@64": 0.999,
+        "pass@128": 0.999
+      },
+      "8": {
+        "pass@1": 0.992984375,
+        "pass@2": 0.9974980314960634,
+        "pass@4": 0.9988455806149232,
+        "pass@8": 0.9992881325051802,
+        "pass@16": 0.9995580380881292,
+        "pass@32": 0.999829071592898,
+        "pass@64": 0.9999861763011332,
+        "pass@128": 1.0
+      },
+      "9": {
+        "pass@1": 0.9869765625,
+        "pass@2": 0.9931916830708665,
+        "pass@4": 0.995589347581552,
+        "pass@8": 0.9970554718258637,
+        "pass@16": 0.9980393105543356,
+        "pass@32": 0.9987247492592555,
+        "pass@64": 0.998984425990599,
+        "pass@128": 0.999
+      },
+      "11": {
+        "pass@1": 0.9569375,
+        "pass@2": 0.9694979084645677,
+        "pass@4": 0.9755786563554552,
+        "pass@8": 0.9795771285286979,
+        "pass@16": 0.9828716597217099,
+        "pass@32": 0.9858994698672985,
+        "pass@64": 0.9886172988906358,
+        "pass@128": 0.991
+      },
+      "12": {
+        "pass@1": 0.9321875,
+        "pass@2": 0.9527299458661426,
+        "pass@4": 0.964783362579677,
+        "pass@8": 0.9725947365272651,
+        "pass@16": 0.9780288578767546,
+        "pass@32": 0.9822927738860049,
+        "pass@64": 0.9860262163530781,
+        "pass@128": 0.989
+      },
+      "13": {
+        "pass@1": 0.9148515625,
+        "pass@2": 0.9376562500000003,
+        "pass@4": 0.9520333217097862,
+        "pass@8": 0.9627994330286813,
+        "pass@16": 0.9713319331348897,
+        "pass@32": 0.9782062595316485,
+        "pass@64": 0.9837086690994824,
+        "pass@128": 0.988
+      },
+      "14": {
+        "pass@1": 0.92659375,
+        "pass@2": 0.9529224901574813,
+        "pass@4": 0.9665270294338203,
+        "pass@8": 0.9751033296649382,
+        "pass@16": 0.9814160863652701,
+        "pass@32": 0.9866717919871326,
+        "pass@64": 0.99098669283729,
+        "pass@128": 0.993
+      },
+      "15": {
+        "pass@1": 0.9321484375,
+        "pass@2": 0.9586374261811037,
+        "pass@4": 0.9716738764529432,
+        "pass@8": 0.980257322244576,
+        "pass@16": 0.9860876075856799,
+        "pass@32": 0.9905182347017089,
+        "pass@64": 0.9941053539635863,
+        "pass@128": 0.997
+      },
+      "16": {
+        "pass@1": 0.919421875,
+        "pass@2": 0.9510446604330718,
+        "pass@4": 0.9672112351893514,
+        "pass@8": 0.9771642191649528,
+        "pass@16": 0.9837104050806683,
+        "pass@32": 0.9884127699664014,
+        "pass@64": 0.9923083437843704,
+        "pass@128": 0.995
+      },
+      "17": {
+        "pass@1": 0.917234375,
+        "pass@2": 0.9551825787401591,
+        "pass@4": 0.9728983112110978,
+        "pass@8": 0.9831834738064922,
+        "pass@16": 0.989503100920689,
+        "pass@32": 0.9936977352294757,
+        "pass@64": 0.9968179233919849,
+        "pass@128": 0.999
+      },
+      "18": {
+        "pass@1": 0.8954921875,
+        "pass@2": 0.9377255167322847,
+        "pass@4": 0.9575529777840263,
+        "pass@8": 0.9690916457538964,
+        "pass@16": 0.9765775044869797,
+        "pass@32": 0.9819170635269723,
+        "pass@64": 0.9859361791907656,
+        "pass@128": 0.989
+      },
+      "19": {
+        "pass@1": 0.875109375,
+        "pass@2": 0.9267666092519707,
+        "pass@4": 0.9520276386389189,
+        "pass@8": 0.9663416476690758,
+        "pass@16": 0.975462333146504,
+        "pass@32": 0.981573341392933,
+        "pass@64": 0.9858043859763082,
+        "pass@128": 0.989
+      },
+      "20": {
+        "pass@1": 0.8575859375,
+        "pass@2": 0.9162995816929158,
+        "pass@4": 0.944188775403074,
+        "pass@8": 0.9605284903730037,
+        "pass@16": 0.9717644090133872,
+        "pass@32": 0.9798880740245233,
+        "pass@64": 0.9855316641548822,
+        "pass@128": 0.989
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.9722734375,
+      "2": 1.0,
+      "3": 0.99696875,
+      "4": 0.998421875,
+      "5": 0.9941015625,
+      "6": 0.99475,
+      "7": 0.9951015625,
+      "8": 0.992984375,
+      "9": 0.9869765625,
+      "11": 0.9569375,
+      "12": 0.9321875,
+      "13": 0.9148515625,
+      "14": 0.92659375,
+      "15": 0.9321484375,
+      "16": 0.919421875,
+      "17": 0.917234375,
+      "18": 0.8954921875,
+      "19": 0.875109375,
+      "20": 0.8575859375
+    },
+    "per_op_avg_response_len": {
+      "10": 245.9643203125,
+      "2": 103.040046875,
+      "3": 121.6964375,
+      "4": 141.7835859375,
+      "5": 164.8098203125,
+      "6": 175.4171015625,
+      "7": 190.5570234375,
+      "8": 213.8619296875,
+      "9": 231.3644296875,
+      "11": 266.557984375,
+      "12": 278.4357265625,
+      "13": 288.8397265625,
+      "14": 317.2391484375,
+      "15": 331.4378515625,
+      "16": 357.8768828125,
+      "17": 368.6404765625,
+      "18": 382.5142890625,
+      "19": 393.4576171875,
+      "20": 410.3684609375
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 245.9643203125,
+      "2": 103.040046875,
+      "3": 121.6964375,
+      "4": 141.7835859375,
+      "5": 164.8098203125,
+      "6": 175.4171015625,
+      "7": 190.5570234375,
+      "8": 213.8619296875,
+      "9": 231.3644296875,
+      "11": 266.557984375,
+      "12": 278.4357265625,
+      "13": 288.8397265625,
+      "14": 317.2391484375,
+      "15": 331.4378515625,
+      "16": 357.8768828125,
+      "17": 368.6404765625,
+      "18": 382.5142890625,
+      "19": 393.4576171875,
+      "20": 410.3684609375
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      },
+      "teachers_in_school": {
+        "count": 2432000,
+        "correct": 2311570,
+        "answer_accuracy": 0.9504810855263158,
+        "avg_response_len": 262.30857154605263,
+        "resp_tokens_sum": 637934446,
+        "pass_at_k": {
+          "pass@1": 0.9504810855263158,
+          "pass@2": 0.9695000518027258,
+          "pass@4": 0.9789090207802891,
+          "pass@8": 0.9846479381669422,
+          "pass@16": 0.988546711162386,
+          "pass@32": 0.9914438404095707,
+          "pass@64": 0.9936642595978096,
+          "pass@128": 0.9952105263157894
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9722734375,
+            "pass@2": 0.982090920275591,
+            "pass@4": 0.9870395954255715,
+            "pass@8": 0.9898463912960546,
+            "pass@16": 0.9913574932933285,
+            "pass@32": 0.9924062358073099,
+            "pass@64": 0.9929435009525102,
+            "pass@128": 0.993
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.99696875,
+            "pass@2": 0.9987176427165356,
+            "pass@4": 0.999073522215973,
+            "pass@8": 0.9991772006158419,
+            "pass@16": 0.9993323490768988,
+            "pass@32": 0.9995814585676791,
+            "pass@64": 0.9998779527559055,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.998421875,
+            "pass@2": 0.999459153543307,
+            "pass@4": 0.9998440072178476,
+            "pass@8": 0.999979701409027,
+            "pass@16": 0.9999997111918157,
+            "pass@32": 0.9999999999851739,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9941015625,
+            "pass@2": 0.9965524114173228,
+            "pass@4": 0.9978783887326584,
+            "pass@8": 0.9987054335254999,
+            "pass@16": 0.9993742678247356,
+            "pass@32": 0.9998140401510567,
+            "pass@64": 0.9999861487158879,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.99475,
+            "pass@2": 0.9969265501968505,
+            "pass@4": 0.9980495649606299,
+            "pass@8": 0.9987365127580474,
+            "pass@16": 0.9989781223272445,
+            "pass@32": 0.9989999037739257,
+            "pass@64": 0.9989999999999666,
+            "pass@128": 0.999
+          },
+          "7": {
+            "pass@1": 0.9951015625,
+            "pass@2": 0.9976016240157483,
+            "pass@4": 0.9984762033183351,
+            "pass@8": 0.9988805544747335,
+            "pass@16": 0.9989943223963219,
+            "pass@32": 0.9989999945303923,
+            "pass@64": 0.999,
+            "pass@128": 0.999
+          },
+          "8": {
+            "pass@1": 0.992984375,
+            "pass@2": 0.9974980314960634,
+            "pass@4": 0.9988455806149232,
+            "pass@8": 0.9992881325051802,
+            "pass@16": 0.9995580380881292,
+            "pass@32": 0.999829071592898,
+            "pass@64": 0.9999861763011332,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9869765625,
+            "pass@2": 0.9931916830708665,
+            "pass@4": 0.995589347581552,
+            "pass@8": 0.9970554718258637,
+            "pass@16": 0.9980393105543356,
+            "pass@32": 0.9987247492592555,
+            "pass@64": 0.998984425990599,
+            "pass@128": 0.999
+          },
+          "11": {
+            "pass@1": 0.9569375,
+            "pass@2": 0.9694979084645677,
+            "pass@4": 0.9755786563554552,
+            "pass@8": 0.9795771285286979,
+            "pass@16": 0.9828716597217099,
+            "pass@32": 0.9858994698672985,
+            "pass@64": 0.9886172988906358,
+            "pass@128": 0.991
+          },
+          "12": {
+            "pass@1": 0.9321875,
+            "pass@2": 0.9527299458661426,
+            "pass@4": 0.964783362579677,
+            "pass@8": 0.9725947365272651,
+            "pass@16": 0.9780288578767546,
+            "pass@32": 0.9822927738860049,
+            "pass@64": 0.9860262163530781,
+            "pass@128": 0.989
+          },
+          "13": {
+            "pass@1": 0.9148515625,
+            "pass@2": 0.9376562500000003,
+            "pass@4": 0.9520333217097862,
+            "pass@8": 0.9627994330286813,
+            "pass@16": 0.9713319331348897,
+            "pass@32": 0.9782062595316485,
+            "pass@64": 0.9837086690994824,
+            "pass@128": 0.988
+          },
+          "14": {
+            "pass@1": 0.92659375,
+            "pass@2": 0.9529224901574813,
+            "pass@4": 0.9665270294338203,
+            "pass@8": 0.9751033296649382,
+            "pass@16": 0.9814160863652701,
+            "pass@32": 0.9866717919871326,
+            "pass@64": 0.99098669283729,
+            "pass@128": 0.993
+          },
+          "15": {
+            "pass@1": 0.9321484375,
+            "pass@2": 0.9586374261811037,
+            "pass@4": 0.9716738764529432,
+            "pass@8": 0.980257322244576,
+            "pass@16": 0.9860876075856799,
+            "pass@32": 0.9905182347017089,
+            "pass@64": 0.9941053539635863,
+            "pass@128": 0.997
+          },
+          "16": {
+            "pass@1": 0.919421875,
+            "pass@2": 0.9510446604330718,
+            "pass@4": 0.9672112351893514,
+            "pass@8": 0.9771642191649528,
+            "pass@16": 0.9837104050806683,
+            "pass@32": 0.9884127699664014,
+            "pass@64": 0.9923083437843704,
+            "pass@128": 0.995
+          },
+          "17": {
+            "pass@1": 0.917234375,
+            "pass@2": 0.9551825787401591,
+            "pass@4": 0.9728983112110978,
+            "pass@8": 0.9831834738064922,
+            "pass@16": 0.989503100920689,
+            "pass@32": 0.9936977352294757,
+            "pass@64": 0.9968179233919849,
+            "pass@128": 0.999
+          },
+          "18": {
+            "pass@1": 0.8954921875,
+            "pass@2": 0.9377255167322847,
+            "pass@4": 0.9575529777840263,
+            "pass@8": 0.9690916457538964,
+            "pass@16": 0.9765775044869797,
+            "pass@32": 0.9819170635269723,
+            "pass@64": 0.9859361791907656,
+            "pass@128": 0.989
+          },
+          "19": {
+            "pass@1": 0.875109375,
+            "pass@2": 0.9267666092519707,
+            "pass@4": 0.9520276386389189,
+            "pass@8": 0.9663416476690758,
+            "pass@16": 0.975462333146504,
+            "pass@32": 0.981573341392933,
+            "pass@64": 0.9858043859763082,
+            "pass@128": 0.989
+          },
+          "20": {
+            "pass@1": 0.8575859375,
+            "pass@2": 0.9162995816929158,
+            "pass@4": 0.944188775403074,
+            "pass@8": 0.9605284903730037,
+            "pass@16": 0.9717644090133872,
+            "pass@32": 0.9798880740245233,
+            "pass@64": 0.9855316641548822,
+            "pass@128": 0.989
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      }
+    }
+  }
+}

Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/hf_model_ood_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaf1178ba74866a661cdbd18ff34a89792dd569d167d8962551e823a762f7bd5
+size 2618033145

Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/summary.csv

@@ -0,0 +1,2 @@
+checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
+hf_model,0.9923975694444445,0.91275625,0.9504810855263158,0.0,0.0,0.0,176.4994105902778,339.53681640625,1152000,1280000,2432000

Qwen3-100M-RL/context/0.9zoo_op2-20+0.1teacher_op2/summary.json

@@ -0,0 +1,245 @@
+[
+  {
+    "checkpoint": "hf_model",
+    "id_acc": 0.9923975694444445,
+    "ood_acc": 0.91275625,
+    "total_acc": 0.9504810855263158,
+    "id_avg_loss": 0.0,
+    "ood_avg_loss": 0.0,
+    "total_avg_loss": 0.0,
+    "id_avg_resp_len": 176.4994105902778,
+    "ood_avg_resp_len": 339.53681640625,
+    "count_id": 1152000,
+    "count_ood": 1280000,
+    "count_total": 2432000,
+    "template_metrics": {
+      "crazy_zootopia": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      },
+      "teachers_in_school": {
+        "count": 2432000,
+        "correct": 2311570,
+        "answer_accuracy": 0.9504810855263158,
+        "avg_response_len": 262.30857154605263,
+        "resp_tokens_sum": 637934446,
+        "pass_at_k": {
+          "pass@1": 0.9504810855263158,
+          "pass@2": 0.9695000518027258,
+          "pass@4": 0.9789090207802891,
+          "pass@8": 0.9846479381669422,
+          "pass@16": 0.988546711162386,
+          "pass@32": 0.9914438404095707,
+          "pass@64": 0.9936642595978096,
+          "pass@128": 0.9952105263157894
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9722734375,
+            "pass@2": 0.982090920275591,
+            "pass@4": 0.9870395954255715,
+            "pass@8": 0.9898463912960546,
+            "pass@16": 0.9913574932933285,
+            "pass@32": 0.9924062358073099,
+            "pass@64": 0.9929435009525102,
+            "pass@128": 0.993
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.99696875,
+            "pass@2": 0.9987176427165356,
+            "pass@4": 0.999073522215973,
+            "pass@8": 0.9991772006158419,
+            "pass@16": 0.9993323490768988,
+            "pass@32": 0.9995814585676791,
+            "pass@64": 0.9998779527559055,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.998421875,
+            "pass@2": 0.999459153543307,
+            "pass@4": 0.9998440072178476,
+            "pass@8": 0.999979701409027,
+            "pass@16": 0.9999997111918157,
+            "pass@32": 0.9999999999851739,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9941015625,
+            "pass@2": 0.9965524114173228,
+            "pass@4": 0.9978783887326584,
+            "pass@8": 0.9987054335254999,
+            "pass@16": 0.9993742678247356,
+            "pass@32": 0.9998140401510567,
+            "pass@64": 0.9999861487158879,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.99475,
+            "pass@2": 0.9969265501968505,
+            "pass@4": 0.9980495649606299,
+            "pass@8": 0.9987365127580474,
+            "pass@16": 0.9989781223272445,
+            "pass@32": 0.9989999037739257,
+            "pass@64": 0.9989999999999666,
+            "pass@128": 0.999
+          },
+          "7": {
+            "pass@1": 0.9951015625,
+            "pass@2": 0.9976016240157483,
+            "pass@4": 0.9984762033183351,
+            "pass@8": 0.9988805544747335,
+            "pass@16": 0.9989943223963219,
+            "pass@32": 0.9989999945303923,
+            "pass@64": 0.999,
+            "pass@128": 0.999
+          },
+          "8": {
+            "pass@1": 0.992984375,
+            "pass@2": 0.9974980314960634,
+            "pass@4": 0.9988455806149232,
+            "pass@8": 0.9992881325051802,
+            "pass@16": 0.9995580380881292,
+            "pass@32": 0.999829071592898,
+            "pass@64": 0.9999861763011332,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9869765625,
+            "pass@2": 0.9931916830708665,
+            "pass@4": 0.995589347581552,
+            "pass@8": 0.9970554718258637,
+            "pass@16": 0.9980393105543356,
+            "pass@32": 0.9987247492592555,
+            "pass@64": 0.998984425990599,
+            "pass@128": 0.999
+          },
+          "11": {
+            "pass@1": 0.9569375,
+            "pass@2": 0.9694979084645677,
+            "pass@4": 0.9755786563554552,
+            "pass@8": 0.9795771285286979,
+            "pass@16": 0.9828716597217099,
+            "pass@32": 0.9858994698672985,
+            "pass@64": 0.9886172988906358,
+            "pass@128": 0.991
+          },
+          "12": {
+            "pass@1": 0.9321875,
+            "pass@2": 0.9527299458661426,
+            "pass@4": 0.964783362579677,
+            "pass@8": 0.9725947365272651,
+            "pass@16": 0.9780288578767546,
+            "pass@32": 0.9822927738860049,
+            "pass@64": 0.9860262163530781,
+            "pass@128": 0.989
+          },
+          "13": {
+            "pass@1": 0.9148515625,
+            "pass@2": 0.9376562500000003,
+            "pass@4": 0.9520333217097862,
+            "pass@8": 0.9627994330286813,
+            "pass@16": 0.9713319331348897,
+            "pass@32": 0.9782062595316485,
+            "pass@64": 0.9837086690994824,
+            "pass@128": 0.988
+          },
+          "14": {
+            "pass@1": 0.92659375,
+            "pass@2": 0.9529224901574813,
+            "pass@4": 0.9665270294338203,
+            "pass@8": 0.9751033296649382,
+            "pass@16": 0.9814160863652701,
+            "pass@32": 0.9866717919871326,
+            "pass@64": 0.99098669283729,
+            "pass@128": 0.993
+          },
+          "15": {
+            "pass@1": 0.9321484375,
+            "pass@2": 0.9586374261811037,
+            "pass@4": 0.9716738764529432,
+            "pass@8": 0.980257322244576,
+            "pass@16": 0.9860876075856799,
+            "pass@32": 0.9905182347017089,
+            "pass@64": 0.9941053539635863,
+            "pass@128": 0.997
+          },
+          "16": {
+            "pass@1": 0.919421875,
+            "pass@2": 0.9510446604330718,
+            "pass@4": 0.9672112351893514,
+            "pass@8": 0.9771642191649528,
+            "pass@16": 0.9837104050806683,
+            "pass@32": 0.9884127699664014,
+            "pass@64": 0.9923083437843704,
+            "pass@128": 0.995
+          },
+          "17": {
+            "pass@1": 0.917234375,
+            "pass@2": 0.9551825787401591,
+            "pass@4": 0.9728983112110978,
+            "pass@8": 0.9831834738064922,
+            "pass@16": 0.989503100920689,
+            "pass@32": 0.9936977352294757,
+            "pass@64": 0.9968179233919849,
+            "pass@128": 0.999
+          },
+          "18": {
+            "pass@1": 0.8954921875,
+            "pass@2": 0.9377255167322847,
+            "pass@4": 0.9575529777840263,
+            "pass@8": 0.9690916457538964,
+            "pass@16": 0.9765775044869797,
+            "pass@32": 0.9819170635269723,
+            "pass@64": 0.9859361791907656,
+            "pass@128": 0.989
+          },
+          "19": {
+            "pass@1": 0.875109375,
+            "pass@2": 0.9267666092519707,
+            "pass@4": 0.9520276386389189,
+            "pass@8": 0.9663416476690758,
+            "pass@16": 0.975462333146504,
+            "pass@32": 0.981573341392933,
+            "pass@64": 0.9858043859763082,
+            "pass@128": 0.989
+          },
+          "20": {
+            "pass@1": 0.8575859375,
+            "pass@2": 0.9162995816929158,
+            "pass@4": 0.944188775403074,
+            "pass@8": 0.9605284903730037,
+            "pass@16": 0.9717644090133872,
+            "pass@32": 0.9798880740245233,
+            "pass@64": 0.9855316641548822,
+            "pass@128": 0.989
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 0,
+        "correct": 0,
+        "answer_accuracy": 0.0,
+        "avg_response_len": 0.0,
+        "resp_tokens_sum": 0,
+        "pass_at_k": {},
+        "per_op_pass_at_k": {}
+      }
+    }
+  }
+]

Qwen3-100M-ep4/composition/checkpoint-85276_id_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fa35f6d2d084c259ecd5f0a8f1e745e8bb7ac8fbd684e1a93afc6cbc6a308b5
+size 1789209154

Qwen3-100M-ep4/composition/checkpoint-85276_metrics.json

@@ -0,0 +1,1897 @@
+{
+  "checkpoint": "checkpoint-85276",
+  "id": {
+    "answer_accuracy": 0.9949305555555555,
+    "avg_response_len": 200.0108559027778,
+    "avg_loss": 0.0,
+    "count": 1152000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.9949305555555555,
+      "pass@2": 0.9962390912073487,
+      "pass@4": 0.9971986516685417,
+      "pass@8": 0.9977962149213019,
+      "pass@16": 0.9981298764933108,
+      "pass@32": 0.9983723358418248,
+      "pass@64": 0.9986656367264329,
+      "pass@128": 0.9991111111111111
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.9624921875,
+        "pass@2": 0.9715636072834645,
+        "pass@4": 0.9781628562054747,
+        "pass@8": 0.9820358637737919,
+        "pass@16": 0.984177149756922,
+        "pass@32": 0.9859330858922759,
+        "pass@64": 0.988238824249548,
+        "pass@128": 0.992
+      },
+      "2": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "3": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "4": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "5": {
+        "pass@1": 0.999453125,
+        "pass@2": 0.9997881397637796,
+        "pass@4": 0.9999573371766028,
+        "pass@8": 0.9999984489981919,
+        "pass@16": 0.9999999988246276,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "6": {
+        "pass@1": 0.9999921875,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "7": {
+        "pass@1": 0.9998984375,
+        "pass@2": 0.9999944635826771,
+        "pass@4": 0.9999999803149605,
+        "pass@8": 0.9999999999999685,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "8": {
+        "pass@1": 0.996578125,
+        "pass@2": 0.9977417568897637,
+        "pass@4": 0.998505216816648,
+        "pass@8": 0.9989649390790856,
+        "pass@16": 0.9992179855257273,
+        "pass@32": 0.9994388347222112,
+        "pass@64": 0.9997519685037116,
+        "pass@128": 1.0
+      },
+      "9": {
+        "pass@1": 0.9959609375,
+        "pass@2": 0.9970638533464568,
+        "pass@4": 0.9981624745031872,
+        "pass@8": 0.9991666824406795,
+        "pass@16": 0.9997737543325198,
+        "pass@32": 0.999979101961939,
+        "pass@64": 0.9999999377846387,
+        "pass@128": 1.0
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.9624921875,
+      "2": 1.0,
+      "3": 1.0,
+      "4": 1.0,
+      "5": 0.999453125,
+      "6": 0.9999921875,
+      "7": 0.9998984375,
+      "8": 0.996578125,
+      "9": 0.9959609375
+    },
+    "per_op_avg_response_len": {
+      "10": 281.153859375,
+      "2": 115.629984375,
+      "3": 144.724578125,
+      "4": 159.8816328125,
+      "5": 172.73553125,
+      "6": 198.9245234375,
+      "7": 223.9551328125,
+      "8": 243.5811875,
+      "9": 259.5112734375
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 281.153859375,
+      "2": 115.629984375,
+      "3": 144.724578125,
+      "4": 159.8816328125,
+      "5": 172.73553125,
+      "6": 198.9245234375,
+      "7": 223.9551328125,
+      "8": 243.5811875,
+      "9": 259.5112734375
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 379008,
+        "correct": 377286,
+        "answer_accuracy": 0.9954565602836879,
+        "avg_response_len": 190.53283571850727,
+        "resp_tokens_sum": 72213469,
+        "pass_at_k": {
+          "pass@1": 0.9954565602836879,
+          "pass@2": 0.9967242292851691,
+          "pass@4": 0.9975758479784755,
+          "pass@8": 0.9980857085222107,
+          "pass@16": 0.9983981190211061,
+          "pass@32": 0.9986417677980687,
+          "pass@64": 0.9989868076836745,
+          "pass@128": 0.999662276258021
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.964855272861357,
+            "pass@2": 0.9744083600445962,
+            "pass@4": 0.980863755415529,
+            "pass@8": 0.9845045622809016,
+            "pass@16": 0.9864817601267433,
+            "pass@32": 0.988192742885706,
+            "pass@64": 0.9911504413060571,
+            "pass@128": 0.9970501474926253
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9985367063492063,
+            "pass@2": 0.99933172415948,
+            "pass@4": 0.999864562762988,
+            "pass@8": 0.999995076184736,
+            "pass@16": 0.999999996268659,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9997688609467456,
+            "pass@2": 0.9999836200670922,
+            "pass@4": 0.9999999417602385,
+            "pass@8": 0.9999999999999069,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.999824719551282,
+            "pass@2": 0.9999976340096912,
+            "pass@4": 0.9999999996995569,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.997337147887324,
+            "pass@2": 0.9977261700121991,
+            "pass@4": 0.9981745219347582,
+            "pass@8": 0.9988346121771267,
+            "pass@16": 0.9995479293349335,
+            "pass@32": 0.9999462946812021,
+            "pass@64": 0.9999998252073432,
+            "pass@128": 1.0
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 390656,
+        "correct": 388437,
+        "answer_accuracy": 0.9943198107798165,
+        "avg_response_len": 198.04131256143512,
+        "resp_tokens_sum": 77366027,
+        "pass_at_k": {
+          "pass@1": 0.9943198107798165,
+          "pass@2": 0.9955572810781106,
+          "pass@4": 0.9964470693825461,
+          "pass@8": 0.9969549085780425,
+          "pass@16": 0.9971560486666131,
+          "pass@32": 0.9972767789698127,
+          "pass@64": 0.9974613265083974,
+          "pass@128": 0.9977064220183486
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9569833205521472,
+            "pass@2": 0.9659142372349164,
+            "pass@4": 0.9719421844700393,
+            "pass@8": 0.9748675318830776,
+            "pass@16": 0.9757861748653367,
+            "pass@32": 0.9762266888126843,
+            "pass@64": 0.9769938650304216,
+            "pass@128": 0.9785276073619632
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9998414855072464,
+            "pass@2": 0.9999967904827115,
+            "pass@4": 0.9999999997282949,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999776146131805,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9948283450704225,
+            "pass@2": 0.9955154153266054,
+            "pass@4": 0.9962789690901313,
+            "pass@8": 0.9971273245981712,
+            "pass@16": 0.997797475275284,
+            "pass@32": 0.9984192527437615,
+            "pass@64": 0.9993013197287653,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9957662799401198,
+            "pass@2": 0.9974428603423076,
+            "pass@4": 0.9988750589372435,
+            "pass@8": 0.9997585789040461,
+            "pass@16": 0.9999876161726882,
+            "pass@32": 0.9999999824517899,
+            "pass@64": 0.9999999999999998,
+            "pass@128": 1.0
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 382336,
+        "correct": 380437,
+        "answer_accuracy": 0.9950331645463676,
+        "avg_response_len": 211.4187782474054,
+        "resp_tokens_sum": 80833010,
+        "pass_at_k": {
+          "pass@1": 0.9950331645463676,
+          "pass@2": 0.9964548229334995,
+          "pass@4": 0.9975926760619602,
+          "pass@8": 0.9983688551648029,
+          "pass@16": 0.9988589881110809,
+          "pass@32": 0.9992246463711005,
+          "pass@64": 0.9995777785346197,
+          "pass@128": 1.0
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9654617537313432,
+            "pass@2": 0.974182483253026,
+            "pass@4": 0.9814832566638125,
+            "pass@8": 0.9865134381095033,
+            "pass@16": 0.990010567366735,
+            "pass@32": 0.9930920761226464,
+            "pass@64": 0.9962352974533647,
+            "pass@128": 1.0
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9999080882352941,
+            "pass@2": 0.9999992762853173,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9999258306962026,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9954016516516516,
+            "pass@2": 0.9980015694828687,
+            "pass@4": 0.9994780266768457,
+            "pass@8": 0.9999541707109151,
+            "pass@16": 0.9999996450540588,
+            "pass@32": 0.9999999999945218,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.994599075562701,
+            "pass@2": 0.9959007962630074,
+            "pass@4": 0.9973834390074231,
+            "pass@8": 0.9989100635493507,
+            "pass@16": 0.9998018508583297,
+            "pass@32": 0.9999941260810753,
+            "pass@64": 0.9999999994727716,
+            "pass@128": 1.0
+          }
+        }
+      }
+    }
+  },
+  "ood": {
+    "answer_accuracy": 0.44683984375,
+    "avg_response_len": 327.42891796875,
+    "avg_loss": 0.0,
+    "count": 1280000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.44683984375,
+      "pass@2": 0.5119684670275582,
+      "pass@4": 0.569463077690281,
+      "pass@8": 0.6178828516789759,
+      "pass@16": 0.6568252672877612,
+      "pass@32": 0.6882615383682829,
+      "pass@64": 0.7149255891967156,
+      "pass@128": 0.7386
+    },
+    "per_op_pass_at_k": {
+      "11": {
+        "pass@1": 0.96253125,
+        "pass@2": 0.972941190944882,
+        "pass@4": 0.9790027408136479,
+        "pass@8": 0.9831148686678516,
+        "pass@16": 0.98574542765598,
+        "pass@32": 0.9875692041741516,
+        "pass@64": 0.9892254753754837,
+        "pass@128": 0.991
+      },
+      "12": {
+        "pass@1": 0.876703125,
+        "pass@2": 0.9067059547244101,
+        "pass@4": 0.9224254388826393,
+        "pass@8": 0.9343966760611316,
+        "pass@16": 0.9450926015777066,
+        "pass@32": 0.9542979874178255,
+        "pass@64": 0.9614691678418349,
+        "pass@128": 0.966
+      },
+      "13": {
+        "pass@1": 0.70778125,
+        "pass@2": 0.7821064222440945,
+        "pass@4": 0.8326372481252345,
+        "pass@8": 0.8670544873404723,
+        "pass@16": 0.890846691933782,
+        "pass@32": 0.9088359604915771,
+        "pass@64": 0.9236104944347704,
+        "pass@128": 0.936
+      },
+      "14": {
+        "pass@1": 0.5578828125,
+        "pass@2": 0.6460127952755912,
+        "pass@4": 0.714011713348332,
+        "pass@8": 0.7653732918196041,
+        "pass@16": 0.803843338086645,
+        "pass@32": 0.8336969542236903,
+        "pass@64": 0.856039598782845,
+        "pass@128": 0.872
+      },
+      "15": {
+        "pass@1": 0.38559375,
+        "pass@2": 0.4773902559055116,
+        "pass@4": 0.5553634880014996,
+        "pass@8": 0.6194323066082491,
+        "pass@16": 0.6699661651256025,
+        "pass@32": 0.7098046818744348,
+        "pass@64": 0.7443967566005082,
+        "pass@128": 0.777
+      },
+      "16": {
+        "pass@1": 0.23978125,
+        "pass@2": 0.3170354330708663,
+        "pass@4": 0.3916617366891629,
+        "pass@8": 0.4572465077324236,
+        "pass@16": 0.5126771573643457,
+        "pass@32": 0.5610609835923893,
+        "pass@64": 0.6070904239009969,
+        "pass@128": 0.651
+      },
+      "17": {
+        "pass@1": 0.2129765625,
+        "pass@2": 0.2896716289370081,
+        "pass@4": 0.3657562349081365,
+        "pass@8": 0.4328031981342337,
+        "pass@16": 0.48777685207242233,
+        "pass@32": 0.5337343412962802,
+        "pass@64": 0.5753223230388924,
+        "pass@128": 0.614
+      },
+      "18": {
+        "pass@1": 0.193296875,
+        "pass@2": 0.2601900836614175,
+        "pass@4": 0.3277885975815519,
+        "pass@8": 0.3916256997651364,
+        "pass@16": 0.4472835877005817,
+        "pass@32": 0.49205269327981904,
+        "pass@64": 0.5271340188878637,
+        "pass@128": 0.556
+      },
+      "19": {
+        "pass@1": 0.1764375,
+        "pass@2": 0.24655782480314958,
+        "pass@4": 0.3155220647731533,
+        "pass@8": 0.3751407394764381,
+        "pass@16": 0.4233042476747667,
+        "pass@32": 0.46136250070818885,
+        "pass@64": 0.4921356285231653,
+        "pass@128": 0.521
+      },
+      "20": {
+        "pass@1": 0.1554140625,
+        "pass@2": 0.22107308070866163,
+        "pass@4": 0.2904615137795272,
+        "pass@8": 0.3526407411842315,
+        "pass@16": 0.40171660368563283,
+        "pass@32": 0.44020007662447264,
+        "pass@64": 0.4728320045807315,
+        "pass@128": 0.502
+      }
+    },
+    "per_op_accuracy": {
+      "11": 0.96253125,
+      "12": 0.876703125,
+      "13": 0.70778125,
+      "14": 0.5578828125,
+      "15": 0.38559375,
+      "16": 0.23978125,
+      "17": 0.2129765625,
+      "18": 0.193296875,
+      "19": 0.1764375,
+      "20": 0.1554140625
+    },
+    "per_op_avg_response_len": {
+      "11": 299.1838515625,
+      "12": 307.2964296875,
+      "13": 318.4115390625,
+      "14": 333.137890625,
+      "15": 335.3022890625,
+      "16": 329.080421875,
+      "17": 338.9223984375,
+      "18": 336.208109375,
+      "19": 336.5952578125,
+      "20": 340.1509921875
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "11": 299.1838515625,
+      "12": 307.2964296875,
+      "13": 318.4115390625,
+      "14": 333.137890625,
+      "15": 335.3022890625,
+      "16": 329.080421875,
+      "17": 338.9223984375,
+      "18": 336.208109375,
+      "19": 336.5952578125,
+      "20": 340.1509921875
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 430720,
+        "correct": 199569,
+        "answer_accuracy": 0.46333813150074293,
+        "avg_response_len": 314.1041906575037,
+        "resp_tokens_sum": 135290957,
+        "pass_at_k": {
+          "pass@1": 0.46333813150074293,
+          "pass@2": 0.5296728934375414,
+          "pass@4": 0.5881637628413112,
+          "pass@8": 0.6371085247462108,
+          "pass@16": 0.6759604016198593,
+          "pass@32": 0.7069346470921534,
+          "pass@64": 0.7328271853390599,
+          "pass@128": 0.7551263001485884
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9568431303116147,
+            "pass@2": 0.966704122147621,
+            "pass@4": 0.9737702782725814,
+            "pass@8": 0.9797293556840062,
+            "pass@16": 0.9839171023985908,
+            "pass@32": 0.9864784015256126,
+            "pass@64": 0.9885864447521222,
+            "pass@128": 0.9915014164305949
+          },
+          "12": {
+            "pass@1": 0.8533501059322034,
+            "pass@2": 0.8813041477601309,
+            "pass@4": 0.8969065471688916,
+            "pass@8": 0.9103012428913164,
+            "pass@16": 0.9232330989994582,
+            "pass@32": 0.9356599220900689,
+            "pass@64": 0.9473396438691338,
+            "pass@128": 0.9548022598870056
+          },
+          "13": {
+            "pass@1": 0.760384012539185,
+            "pass@2": 0.8294336077308518,
+            "pass@4": 0.874541486977137,
+            "pass@8": 0.9027689973847379,
+            "pass@16": 0.9198466852847369,
+            "pass@32": 0.9308948171804944,
+            "pass@64": 0.9399058726440269,
+            "pass@128": 0.9498432601880877
+          },
+          "14": {
+            "pass@1": 0.6034852024922118,
+            "pass@2": 0.6929789259695341,
+            "pass@4": 0.7630072742075453,
+            "pass@8": 0.8164932601872855,
+            "pass@16": 0.8526563967709537,
+            "pass@32": 0.8763087593659933,
+            "pass@64": 0.8936422032813724,
+            "pass@128": 0.9065420560747663
+          },
+          "15": {
+            "pass@1": 0.4252619219653179,
+            "pass@2": 0.5253370209594465,
+            "pass@4": 0.6099591160209015,
+            "pass@8": 0.6784789856366449,
+            "pass@16": 0.7314473595873143,
+            "pass@32": 0.7713284427298922,
+            "pass@64": 0.8031345386516523,
+            "pass@128": 0.8294797687861272
+          },
+          "16": {
+            "pass@1": 0.26829637096774195,
+            "pass@2": 0.3590876619253239,
+            "pass@4": 0.44438377874136703,
+            "pass@8": 0.5160205150620084,
+            "pass@16": 0.572072708861716,
+            "pass@32": 0.6173416875891375,
+            "pass@64": 0.6591309169128403,
+            "pass@128": 0.6967741935483871
+          },
+          "17": {
+            "pass@1": 0.210305274566474,
+            "pass@2": 0.2880021534067633,
+            "pass@4": 0.36638274885075767,
+            "pass@8": 0.4354115772958514,
+            "pass@16": 0.4924820483803051,
+            "pass@32": 0.5414044485040482,
+            "pass@64": 0.5843553236837202,
+            "pass@128": 0.6242774566473989
+          },
+          "18": {
+            "pass@1": 0.19066964285714286,
+            "pass@2": 0.2540649606299211,
+            "pass@4": 0.3153792696448659,
+            "pass@8": 0.3744374956178368,
+            "pass@16": 0.4304686242139373,
+            "pass@32": 0.4799023237679476,
+            "pass@64": 0.5191885486106175,
+            "pass@128": 0.5485714285714286
+          },
+          "19": {
+            "pass@1": 0.17203922672672672,
+            "pass@2": 0.2452760339079237,
+            "pass@4": 0.31713847114831356,
+            "pass@8": 0.3807029920118585,
+            "pass@16": 0.43294372077217796,
+            "pass@32": 0.47306262676523364,
+            "pass@64": 0.5035309613007347,
+            "pass@128": 0.5315315315315315
+          },
+          "20": {
+            "pass@1": 0.16786317567567569,
+            "pass@2": 0.23649646201319421,
+            "pass@4": 0.3076818460755471,
+            "pass@8": 0.3678730399252492,
+            "pass@16": 0.41342134802722214,
+            "pass@32": 0.450621229184501,
+            "pass@64": 0.48393762875113866,
+            "pass@128": 0.5135135135135135
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 430464,
+        "correct": 189989,
+        "answer_accuracy": 0.4413586269699673,
+        "avg_response_len": 322.06403787540887,
+        "resp_tokens_sum": 138636974,
+        "pass_at_k": {
+          "pass@1": 0.4413586269699673,
+          "pass@2": 0.5069858329762768,
+          "pass@4": 0.5649965586928776,
+          "pass@8": 0.6138581833337501,
+          "pass@16": 0.6533552733745646,
+          "pass@32": 0.6853306195144797,
+          "pass@64": 0.7123738268599717,
+          "pass@128": 0.736544751709783
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9643731174698795,
+            "pass@2": 0.9750883455080163,
+            "pass@4": 0.9803072865138847,
+            "pass@8": 0.9823661486047269,
+            "pass@16": 0.9832962407555205,
+            "pass@32": 0.9844320989052803,
+            "pass@64": 0.9860781709515227,
+            "pass@128": 0.9879518072289156
+          },
+          "12": {
+            "pass@1": 0.8863636363636364,
+            "pass@2": 0.9133642287660749,
+            "pass@4": 0.9258904760729354,
+            "pass@8": 0.9354300779654707,
+            "pass@16": 0.945221377305145,
+            "pass@32": 0.9540520042583904,
+            "pass@64": 0.9598380142135106,
+            "pass@128": 0.9623824451410659
+          },
+          "13": {
+            "pass@1": 0.6867997198879552,
+            "pass@2": 0.7681042397273864,
+            "pass@4": 0.824483724986057,
+            "pass@8": 0.8627893286369971,
+            "pass@16": 0.8897924787212615,
+            "pass@32": 0.9118141519741789,
+            "pass@64": 0.9305813041850224,
+            "pass@128": 0.9439775910364145
+          },
+          "14": {
+            "pass@1": 0.5445906432748538,
+            "pass@2": 0.6282847970483952,
+            "pass@4": 0.6919824232497251,
+            "pass@8": 0.7401227434606403,
+            "pass@16": 0.7772650766591268,
+            "pass@32": 0.8067142511689581,
+            "pass@64": 0.8289431092869494,
+            "pass@128": 0.8450292397660819
+          },
+          "15": {
+            "pass@1": 0.3677556818181818,
+            "pass@2": 0.4612525351944643,
+            "pass@4": 0.5395015228210107,
+            "pass@8": 0.6038984673310392,
+            "pass@16": 0.6548475782041733,
+            "pass@32": 0.695211253240757,
+            "pass@64": 0.7307349783554408,
+            "pass@128": 0.7636363636363637
+          },
+          "16": {
+            "pass@1": 0.23595530626780628,
+            "pass@2": 0.30808173217129914,
+            "pass@4": 0.37879288913672104,
+            "pass@8": 0.44199327843430775,
+            "pass@16": 0.49717687261406224,
+            "pass@32": 0.5475518095537767,
+            "pass@64": 0.5956511172414417,
+            "pass@128": 0.6410256410256411
+          },
+          "17": {
+            "pass@1": 0.23090547360248448,
+            "pass@2": 0.31294512948109743,
+            "pass@4": 0.3923752244478753,
+            "pass@8": 0.4620048777355182,
+            "pass@16": 0.5184937076070995,
+            "pass@32": 0.5641996443460395,
+            "pass@64": 0.6058905683329744,
+            "pass@128": 0.6490683229813664
+          },
+          "18": {
+            "pass@1": 0.19181985294117648,
+            "pass@2": 0.2639264416396479,
+            "pass@4": 0.3402037715873749,
+            "pass@8": 0.41292313703410055,
+            "pass@16": 0.4755867515992984,
+            "pass@32": 0.5242663505584598,
+            "pass@64": 0.5607374408176437,
+            "pass@128": 0.5911764705882353
+          },
+          "19": {
+            "pass@1": 0.17740885416666666,
+            "pass@2": 0.24643025578833894,
+            "pass@4": 0.3146818307867766,
+            "pass@8": 0.37225657621985075,
+            "pass@16": 0.4180985090398595,
+            "pass@32": 0.4553241566350863,
+            "pass@64": 0.48681976016382017,
+            "pass@128": 0.5178571428571429
+          },
+          "20": {
+            "pass@1": 0.1394320733532934,
+            "pass@2": 0.2010787725729644,
+            "pass@4": 0.26803790703183067,
+            "pass@8": 0.32950827537338695,
+            "pass@16": 0.37733200724855503,
+            "pass@32": 0.4121459313058152,
+            "pass@64": 0.43980782684006975,
+            "pass@128": 0.46407185628742514
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 418816,
+        "correct": 182397,
+        "answer_accuracy": 0.4355062843826406,
+        "avg_response_len": 346.6464604981663,
+        "resp_tokens_sum": 145181084,
+        "pass_at_k": {
+          "pass@1": 0.4355062843826406,
+          "pass@2": 0.49888203776254814,
+          "pass@4": 0.5548216039296309,
+          "pass@8": 0.6022473289936959,
+          "pass@16": 0.6407127558276247,
+          "pass@32": 0.6720701169897646,
+          "pass@64": 0.6991379075706123,
+          "pass@128": 0.7237163814180929
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9669642857142857,
+            "pass@2": 0.977667635295588,
+            "pass@4": 0.9834914713041819,
+            "pass@8": 0.987697919919454,
+            "pass@16": 0.9903756780268099,
+            "pass@32": 0.9920980050763724,
+            "pass@64": 0.9932587543558059,
+            "pass@128": 0.9936507936507937
+          },
+          "12": {
+            "pass@1": 0.8925602064220184,
+            "pass@2": 0.927709778106865,
+            "pass@4": 0.9466711294115757,
+            "pass@8": 0.9594735205095442,
+            "pass@16": 0.968631391961948,
+            "pass@32": 0.9747149407936225,
+            "pass@64": 0.9783565974863963,
+            "pass@128": 0.981651376146789
+          },
+          "13": {
+            "pass@1": 0.6791087962962963,
+            "pass@2": 0.7509379252940609,
+            "pass@4": 0.8003636541960031,
+            "pass@8": 0.836590700158435,
+            "pass@16": 0.8634558161248805,
+            "pass@32": 0.8838359924574606,
+            "pass@64": 0.8998857884792347,
+            "pass@128": 0.9135802469135802
+          },
+          "14": {
+            "pass@1": 0.5279349035608308,
+            "pass@2": 0.6192675354564355,
+            "pass@4": 0.6896984853896232,
+            "pass@8": 0.7423055698396044,
+            "pass@16": 0.7843204109962838,
+            "pass@32": 0.8204913013870107,
+            "pass@64": 0.8477207957073809,
+            "pass@128": 0.8664688427299704
+          },
+          "15": {
+            "pass@1": 0.36140046296296297,
+            "pass@2": 0.4426244137017595,
+            "pass@4": 0.5132164547139938,
+            "pass@8": 0.5721977881442194,
+            "pass@16": 0.6197088824105395,
+            "pass@32": 0.658967059013711,
+            "pass@64": 0.6955853807708048,
+            "pass@128": 0.7345679012345679
+          },
+          "16": {
+            "pass@1": 0.21766685103244837,
+            "pass@2": 0.2878512385896452,
+            "pass@4": 0.35677422180634516,
+            "pass@8": 0.4192935319550409,
+            "pass@16": 0.4744116086421201,
+            "pass@32": 0.5235822279834264,
+            "pass@64": 0.5713460103429768,
+            "pass@128": 0.6194690265486725
+          },
+          "17": {
+            "pass@1": 0.19837161144578314,
+            "pass@2": 0.26883901254624787,
+            "pass@4": 0.33928608895650064,
+            "pass@8": 0.40176271614166426,
+            "pass@16": 0.4530815948293738,
+            "pass@32": 0.49619312239293656,
+            "pass@64": 0.536260897714176,
+            "pass@128": 0.5692771084337349
+          },
+          "18": {
+            "pass@1": 0.19788306451612903,
+            "pass@2": 0.2630076041402083,
+            "pass@4": 0.3281824866649735,
+            "pass@8": 0.38767325712032175,
+            "pass@16": 0.4352260441352987,
+            "pass@32": 0.47043974442309966,
+            "pass@64": 0.4992493451488665,
+            "pass@128": 0.5258064516129032
+          },
+          "19": {
+            "pass@1": 0.17987632175226587,
+            "pass@2": 0.247976856697671,
+            "pass@4": 0.3147488179347673,
+            "pass@8": 0.37247260884174876,
+            "pass@16": 0.41889090519709676,
+            "pass@32": 0.45572123675521775,
+            "pass@64": 0.4860676102567291,
+            "pass@128": 0.513595166163142
+          },
+          "20": {
+            "pass@1": 0.15899493243243243,
+            "pass@2": 0.2257040505071999,
+            "pass@4": 0.29573212638960666,
+            "pass@8": 0.3606103751183562,
+            "pass@16": 0.41446968279744983,
+            "pass@32": 0.457917316065741,
+            "pass@64": 0.49484972985591286,
+            "pass@128": 0.5285285285285285
+          }
+        }
+      }
+    }
+  },
+  "total": {
+    "answer_accuracy": 0.7064617598684211,
+    "avg_loss": 0.0,
+    "count": 2432000,
+    "validation_metrics": {
+      "id": {},
+      "ood": {}
+    },
+    "pass_at_k": {
+      "pass@1": 0.7064617598684211,
+      "pass@2": 0.7413598153232354,
+      "pass@4": 0.7720746653641805,
+      "pass@8": 0.7978418132148272,
+      "pass@16": 0.8184958716482886,
+      "pass@32": 0.835156126645237,
+      "pass@64": 0.8493287696055289,
+      "pass@128": 0.862
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.9624921875,
+        "pass@2": 0.9715636072834645,
+        "pass@4": 0.9781628562054747,
+        "pass@8": 0.9820358637737919,
+        "pass@16": 0.984177149756922,
+        "pass@32": 0.9859330858922759,
+        "pass@64": 0.988238824249548,
+        "pass@128": 0.992
+      },
+      "2": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "3": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "4": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "5": {
+        "pass@1": 0.999453125,
+        "pass@2": 0.9997881397637796,
+        "pass@4": 0.9999573371766028,
+        "pass@8": 0.9999984489981919,
+        "pass@16": 0.9999999988246276,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "6": {
+        "pass@1": 0.9999921875,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "7": {
+        "pass@1": 0.9998984375,
+        "pass@2": 0.9999944635826771,
+        "pass@4": 0.9999999803149605,
+        "pass@8": 0.9999999999999685,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "8": {
+        "pass@1": 0.996578125,
+        "pass@2": 0.9977417568897637,
+        "pass@4": 0.998505216816648,
+        "pass@8": 0.9989649390790856,
+        "pass@16": 0.9992179855257273,
+        "pass@32": 0.9994388347222112,
+        "pass@64": 0.9997519685037116,
+        "pass@128": 1.0
+      },
+      "9": {
+        "pass@1": 0.9959609375,
+        "pass@2": 0.9970638533464568,
+        "pass@4": 0.9981624745031872,
+        "pass@8": 0.9991666824406795,
+        "pass@16": 0.9997737543325198,
+        "pass@32": 0.999979101961939,
+        "pass@64": 0.9999999377846387,
+        "pass@128": 1.0
+      },
+      "11": {
+        "pass@1": 0.96253125,
+        "pass@2": 0.972941190944882,
+        "pass@4": 0.9790027408136479,
+        "pass@8": 0.9831148686678516,
+        "pass@16": 0.98574542765598,
+        "pass@32": 0.9875692041741516,
+        "pass@64": 0.9892254753754837,
+        "pass@128": 0.991
+      },
+      "12": {
+        "pass@1": 0.876703125,
+        "pass@2": 0.9067059547244101,
+        "pass@4": 0.9224254388826393,
+        "pass@8": 0.9343966760611316,
+        "pass@16": 0.9450926015777066,
+        "pass@32": 0.9542979874178255,
+        "pass@64": 0.9614691678418349,
+        "pass@128": 0.966
+      },
+      "13": {
+        "pass@1": 0.70778125,
+        "pass@2": 0.7821064222440945,
+        "pass@4": 0.8326372481252345,
+        "pass@8": 0.8670544873404723,
+        "pass@16": 0.890846691933782,
+        "pass@32": 0.9088359604915771,
+        "pass@64": 0.9236104944347704,
+        "pass@128": 0.936
+      },
+      "14": {
+        "pass@1": 0.5578828125,
+        "pass@2": 0.6460127952755912,
+        "pass@4": 0.714011713348332,
+        "pass@8": 0.7653732918196041,
+        "pass@16": 0.803843338086645,
+        "pass@32": 0.8336969542236903,
+        "pass@64": 0.856039598782845,
+        "pass@128": 0.872
+      },
+      "15": {
+        "pass@1": 0.38559375,
+        "pass@2": 0.4773902559055116,
+        "pass@4": 0.5553634880014996,
+        "pass@8": 0.6194323066082491,
+        "pass@16": 0.6699661651256025,
+        "pass@32": 0.7098046818744348,
+        "pass@64": 0.7443967566005082,
+        "pass@128": 0.777
+      },
+      "16": {
+        "pass@1": 0.23978125,
+        "pass@2": 0.3170354330708663,
+        "pass@4": 0.3916617366891629,
+        "pass@8": 0.4572465077324236,
+        "pass@16": 0.5126771573643457,
+        "pass@32": 0.5610609835923893,
+        "pass@64": 0.6070904239009969,
+        "pass@128": 0.651
+      },
+      "17": {
+        "pass@1": 0.2129765625,
+        "pass@2": 0.2896716289370081,
+        "pass@4": 0.3657562349081365,
+        "pass@8": 0.4328031981342337,
+        "pass@16": 0.48777685207242233,
+        "pass@32": 0.5337343412962802,
+        "pass@64": 0.5753223230388924,
+        "pass@128": 0.614
+      },
+      "18": {
+        "pass@1": 0.193296875,
+        "pass@2": 0.2601900836614175,
+        "pass@4": 0.3277885975815519,
+        "pass@8": 0.3916256997651364,
+        "pass@16": 0.4472835877005817,
+        "pass@32": 0.49205269327981904,
+        "pass@64": 0.5271340188878637,
+        "pass@128": 0.556
+      },
+      "19": {
+        "pass@1": 0.1764375,
+        "pass@2": 0.24655782480314958,
+        "pass@4": 0.3155220647731533,
+        "pass@8": 0.3751407394764381,
+        "pass@16": 0.4233042476747667,
+        "pass@32": 0.46136250070818885,
+        "pass@64": 0.4921356285231653,
+        "pass@128": 0.521
+      },
+      "20": {
+        "pass@1": 0.1554140625,
+        "pass@2": 0.22107308070866163,
+        "pass@4": 0.2904615137795272,
+        "pass@8": 0.3526407411842315,
+        "pass@16": 0.40171660368563283,
+        "pass@32": 0.44020007662447264,
+        "pass@64": 0.4728320045807315,
+        "pass@128": 0.502
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.9624921875,
+      "2": 1.0,
+      "3": 1.0,
+      "4": 1.0,
+      "5": 0.999453125,
+      "6": 0.9999921875,
+      "7": 0.9998984375,
+      "8": 0.996578125,
+      "9": 0.9959609375,
+      "11": 0.96253125,
+      "12": 0.876703125,
+      "13": 0.70778125,
+      "14": 0.5578828125,
+      "15": 0.38559375,
+      "16": 0.23978125,
+      "17": 0.2129765625,
+      "18": 0.193296875,
+      "19": 0.1764375,
+      "20": 0.1554140625
+    },
+    "per_op_avg_response_len": {
+      "10": 281.153859375,
+      "2": 115.629984375,
+      "3": 144.724578125,
+      "4": 159.8816328125,
+      "5": 172.73553125,
+      "6": 198.9245234375,
+      "7": 223.9551328125,
+      "8": 243.5811875,
+      "9": 259.5112734375,
+      "11": 299.1838515625,
+      "12": 307.2964296875,
+      "13": 318.4115390625,
+      "14": 333.137890625,
+      "15": 335.3022890625,
+      "16": 329.080421875,
+      "17": 338.9223984375,
+      "18": 336.208109375,
+      "19": 336.5952578125,
+      "20": 340.1509921875
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 281.153859375,
+      "2": 115.629984375,
+      "3": 144.724578125,
+      "4": 159.8816328125,
+      "5": 172.73553125,
+      "6": 198.9245234375,
+      "7": 223.9551328125,
+      "8": 243.5811875,
+      "9": 259.5112734375,
+      "11": 299.1838515625,
+      "12": 307.2964296875,
+      "13": 318.4115390625,
+      "14": 333.137890625,
+      "15": 335.3022890625,
+      "16": 329.080421875,
+      "17": 338.9223984375,
+      "18": 336.208109375,
+      "19": 336.5952578125,
+      "20": 340.1509921875
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 809728,
+        "correct": 576855,
+        "answer_accuracy": 0.7124058943250079,
+        "avg_response_len": 256.26435790784063,
+        "resp_tokens_sum": 207504426,
+        "pass_at_k": {
+          "pass@1": 0.7124058943250079,
+          "pass@2": 0.7482848133624225,
+          "pass@4": 0.7797965772724061,
+          "pass@8": 0.8060704977403156,
+          "pass@16": 0.8268832725059063,
+          "pass@32": 0.8434735001446694,
+          "pass@64": 0.8574080645300883,
+          "pass@128": 0.8695858362314258
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.964855272861357,
+            "pass@2": 0.9744083600445962,
+            "pass@4": 0.980863755415529,
+            "pass@8": 0.9845045622809016,
+            "pass@16": 0.9864817601267433,
+            "pass@32": 0.988192742885706,
+            "pass@64": 0.9911504413060571,
+            "pass@128": 0.9970501474926253
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9985367063492063,
+            "pass@2": 0.99933172415948,
+            "pass@4": 0.999864562762988,
+            "pass@8": 0.999995076184736,
+            "pass@16": 0.999999996268659,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9997688609467456,
+            "pass@2": 0.9999836200670922,
+            "pass@4": 0.9999999417602385,
+            "pass@8": 0.9999999999999069,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.999824719551282,
+            "pass@2": 0.9999976340096912,
+            "pass@4": 0.9999999996995569,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.997337147887324,
+            "pass@2": 0.9977261700121991,
+            "pass@4": 0.9981745219347582,
+            "pass@8": 0.9988346121771267,
+            "pass@16": 0.9995479293349335,
+            "pass@32": 0.9999462946812021,
+            "pass@64": 0.9999998252073432,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9568431303116147,
+            "pass@2": 0.966704122147621,
+            "pass@4": 0.9737702782725814,
+            "pass@8": 0.9797293556840062,
+            "pass@16": 0.9839171023985908,
+            "pass@32": 0.9864784015256126,
+            "pass@64": 0.9885864447521222,
+            "pass@128": 0.9915014164305949
+          },
+          "12": {
+            "pass@1": 0.8533501059322034,
+            "pass@2": 0.8813041477601309,
+            "pass@4": 0.8969065471688916,
+            "pass@8": 0.9103012428913164,
+            "pass@16": 0.9232330989994582,
+            "pass@32": 0.9356599220900689,
+            "pass@64": 0.9473396438691338,
+            "pass@128": 0.9548022598870056
+          },
+          "13": {
+            "pass@1": 0.760384012539185,
+            "pass@2": 0.8294336077308518,
+            "pass@4": 0.874541486977137,
+            "pass@8": 0.9027689973847379,
+            "pass@16": 0.9198466852847369,
+            "pass@32": 0.9308948171804944,
+            "pass@64": 0.9399058726440269,
+            "pass@128": 0.9498432601880877
+          },
+          "14": {
+            "pass@1": 0.6034852024922118,
+            "pass@2": 0.6929789259695341,
+            "pass@4": 0.7630072742075453,
+            "pass@8": 0.8164932601872855,
+            "pass@16": 0.8526563967709537,
+            "pass@32": 0.8763087593659933,
+            "pass@64": 0.8936422032813724,
+            "pass@128": 0.9065420560747663
+          },
+          "15": {
+            "pass@1": 0.4252619219653179,
+            "pass@2": 0.5253370209594465,
+            "pass@4": 0.6099591160209015,
+            "pass@8": 0.6784789856366449,
+            "pass@16": 0.7314473595873143,
+            "pass@32": 0.7713284427298922,
+            "pass@64": 0.8031345386516523,
+            "pass@128": 0.8294797687861272
+          },
+          "16": {
+            "pass@1": 0.26829637096774195,
+            "pass@2": 0.3590876619253239,
+            "pass@4": 0.44438377874136703,
+            "pass@8": 0.5160205150620084,
+            "pass@16": 0.572072708861716,
+            "pass@32": 0.6173416875891375,
+            "pass@64": 0.6591309169128403,
+            "pass@128": 0.6967741935483871
+          },
+          "17": {
+            "pass@1": 0.210305274566474,
+            "pass@2": 0.2880021534067633,
+            "pass@4": 0.36638274885075767,
+            "pass@8": 0.4354115772958514,
+            "pass@16": 0.4924820483803051,
+            "pass@32": 0.5414044485040482,
+            "pass@64": 0.5843553236837202,
+            "pass@128": 0.6242774566473989
+          },
+          "18": {
+            "pass@1": 0.19066964285714286,
+            "pass@2": 0.2540649606299211,
+            "pass@4": 0.3153792696448659,
+            "pass@8": 0.3744374956178368,
+            "pass@16": 0.4304686242139373,
+            "pass@32": 0.4799023237679476,
+            "pass@64": 0.5191885486106175,
+            "pass@128": 0.5485714285714286
+          },
+          "19": {
+            "pass@1": 0.17203922672672672,
+            "pass@2": 0.2452760339079237,
+            "pass@4": 0.31713847114831356,
+            "pass@8": 0.3807029920118585,
+            "pass@16": 0.43294372077217796,
+            "pass@32": 0.47306262676523364,
+            "pass@64": 0.5035309613007347,
+            "pass@128": 0.5315315315315315
+          },
+          "20": {
+            "pass@1": 0.16786317567567569,
+            "pass@2": 0.23649646201319421,
+            "pass@4": 0.3076818460755471,
+            "pass@8": 0.3678730399252492,
+            "pass@16": 0.41342134802722214,
+            "pass@32": 0.450621229184501,
+            "pass@64": 0.48393762875113866,
+            "pass@128": 0.5135135135135135
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 821120,
+        "correct": 578426,
+        "answer_accuracy": 0.704435405300078,
+        "avg_response_len": 263.05899381332813,
+        "resp_tokens_sum": 216003001,
+        "pass_at_k": {
+          "pass@1": 0.704435405300078,
+          "pass@2": 0.7394285546608842,
+          "pass@4": 0.7702634267559836,
+          "pass@8": 0.7961202574484137,
+          "pass@16": 0.8169219087902136,
+          "pass@32": 0.8337421048859038,
+          "pass@64": 0.8480070379164071,
+          "pass@128": 0.8607950116913484
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9569833205521472,
+            "pass@2": 0.9659142372349164,
+            "pass@4": 0.9719421844700393,
+            "pass@8": 0.9748675318830776,
+            "pass@16": 0.9757861748653367,
+            "pass@32": 0.9762266888126843,
+            "pass@64": 0.9769938650304216,
+            "pass@128": 0.9785276073619632
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9998414855072464,
+            "pass@2": 0.9999967904827115,
+            "pass@4": 0.9999999997282949,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999776146131805,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9948283450704225,
+            "pass@2": 0.9955154153266054,
+            "pass@4": 0.9962789690901313,
+            "pass@8": 0.9971273245981712,
+            "pass@16": 0.997797475275284,
+            "pass@32": 0.9984192527437615,
+            "pass@64": 0.9993013197287653,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9957662799401198,
+            "pass@2": 0.9974428603423076,
+            "pass@4": 0.9988750589372435,
+            "pass@8": 0.9997585789040461,
+            "pass@16": 0.9999876161726882,
+            "pass@32": 0.9999999824517899,
+            "pass@64": 0.9999999999999998,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9643731174698795,
+            "pass@2": 0.9750883455080163,
+            "pass@4": 0.9803072865138847,
+            "pass@8": 0.9823661486047269,
+            "pass@16": 0.9832962407555205,
+            "pass@32": 0.9844320989052803,
+            "pass@64": 0.9860781709515227,
+            "pass@128": 0.9879518072289156
+          },
+          "12": {
+            "pass@1": 0.8863636363636364,
+            "pass@2": 0.9133642287660749,
+            "pass@4": 0.9258904760729354,
+            "pass@8": 0.9354300779654707,
+            "pass@16": 0.945221377305145,
+            "pass@32": 0.9540520042583904,
+            "pass@64": 0.9598380142135106,
+            "pass@128": 0.9623824451410659
+          },
+          "13": {
+            "pass@1": 0.6867997198879552,
+            "pass@2": 0.7681042397273864,
+            "pass@4": 0.824483724986057,
+            "pass@8": 0.8627893286369971,
+            "pass@16": 0.8897924787212615,
+            "pass@32": 0.9118141519741789,
+            "pass@64": 0.9305813041850224,
+            "pass@128": 0.9439775910364145
+          },
+          "14": {
+            "pass@1": 0.5445906432748538,
+            "pass@2": 0.6282847970483952,
+            "pass@4": 0.6919824232497251,
+            "pass@8": 0.7401227434606403,
+            "pass@16": 0.7772650766591268,
+            "pass@32": 0.8067142511689581,
+            "pass@64": 0.8289431092869494,
+            "pass@128": 0.8450292397660819
+          },
+          "15": {
+            "pass@1": 0.3677556818181818,
+            "pass@2": 0.4612525351944643,
+            "pass@4": 0.5395015228210107,
+            "pass@8": 0.6038984673310392,
+            "pass@16": 0.6548475782041733,
+            "pass@32": 0.695211253240757,
+            "pass@64": 0.7307349783554408,
+            "pass@128": 0.7636363636363637
+          },
+          "16": {
+            "pass@1": 0.23595530626780628,
+            "pass@2": 0.30808173217129914,
+            "pass@4": 0.37879288913672104,
+            "pass@8": 0.44199327843430775,
+            "pass@16": 0.49717687261406224,
+            "pass@32": 0.5475518095537767,
+            "pass@64": 0.5956511172414417,
+            "pass@128": 0.6410256410256411
+          },
+          "17": {
+            "pass@1": 0.23090547360248448,
+            "pass@2": 0.31294512948109743,
+            "pass@4": 0.3923752244478753,
+            "pass@8": 0.4620048777355182,
+            "pass@16": 0.5184937076070995,
+            "pass@32": 0.5641996443460395,
+            "pass@64": 0.6058905683329744,
+            "pass@128": 0.6490683229813664
+          },
+          "18": {
+            "pass@1": 0.19181985294117648,
+            "pass@2": 0.2639264416396479,
+            "pass@4": 0.3402037715873749,
+            "pass@8": 0.41292313703410055,
+            "pass@16": 0.4755867515992984,
+            "pass@32": 0.5242663505584598,
+            "pass@64": 0.5607374408176437,
+            "pass@128": 0.5911764705882353
+          },
+          "19": {
+            "pass@1": 0.17740885416666666,
+            "pass@2": 0.24643025578833894,
+            "pass@4": 0.3146818307867766,
+            "pass@8": 0.37225657621985075,
+            "pass@16": 0.4180985090398595,
+            "pass@32": 0.4553241566350863,
+            "pass@64": 0.48681976016382017,
+            "pass@128": 0.5178571428571429
+          },
+          "20": {
+            "pass@1": 0.1394320733532934,
+            "pass@2": 0.2010787725729644,
+            "pass@4": 0.26803790703183067,
+            "pass@8": 0.32950827537338695,
+            "pass@16": 0.37733200724855503,
+            "pass@32": 0.4121459313058152,
+            "pass@64": 0.43980782684006975,
+            "pass@128": 0.46407185628742514
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 801152,
+        "correct": 562834,
+        "answer_accuracy": 0.7025308555679821,
+        "avg_response_len": 282.1113771169516,
+        "resp_tokens_sum": 226014094,
+        "pass_at_k": {
+          "pass@1": 0.7025308555679821,
+          "pass@2": 0.736340083665345,
+          "pass@4": 0.7661264757077474,
+          "pass@8": 0.791289508043559,
+          "pass@16": 0.8116318796222768,
+          "pass@32": 0.8281989841030529,
+          "pass@64": 0.8425176638526883,
+          "pass@128": 0.8555679821057677
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9654617537313432,
+            "pass@2": 0.974182483253026,
+            "pass@4": 0.9814832566638125,
+            "pass@8": 0.9865134381095033,
+            "pass@16": 0.990010567366735,
+            "pass@32": 0.9930920761226464,
+            "pass@64": 0.9962352974533647,
+            "pass@128": 1.0
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9999080882352941,
+            "pass@2": 0.9999992762853173,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9999258306962026,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9954016516516516,
+            "pass@2": 0.9980015694828687,
+            "pass@4": 0.9994780266768457,
+            "pass@8": 0.9999541707109151,
+            "pass@16": 0.9999996450540588,
+            "pass@32": 0.9999999999945218,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.994599075562701,
+            "pass@2": 0.9959007962630074,
+            "pass@4": 0.9973834390074231,
+            "pass@8": 0.9989100635493507,
+            "pass@16": 0.9998018508583297,
+            "pass@32": 0.9999941260810753,
+            "pass@64": 0.9999999994727716,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9669642857142857,
+            "pass@2": 0.977667635295588,
+            "pass@4": 0.9834914713041819,
+            "pass@8": 0.987697919919454,
+            "pass@16": 0.9903756780268099,
+            "pass@32": 0.9920980050763724,
+            "pass@64": 0.9932587543558059,
+            "pass@128": 0.9936507936507937
+          },
+          "12": {
+            "pass@1": 0.8925602064220184,
+            "pass@2": 0.927709778106865,
+            "pass@4": 0.9466711294115757,
+            "pass@8": 0.9594735205095442,
+            "pass@16": 0.968631391961948,
+            "pass@32": 0.9747149407936225,
+            "pass@64": 0.9783565974863963,
+            "pass@128": 0.981651376146789
+          },
+          "13": {
+            "pass@1": 0.6791087962962963,
+            "pass@2": 0.7509379252940609,
+            "pass@4": 0.8003636541960031,
+            "pass@8": 0.836590700158435,
+            "pass@16": 0.8634558161248805,
+            "pass@32": 0.8838359924574606,
+            "pass@64": 0.8998857884792347,
+            "pass@128": 0.9135802469135802
+          },
+          "14": {
+            "pass@1": 0.5279349035608308,
+            "pass@2": 0.6192675354564355,
+            "pass@4": 0.6896984853896232,
+            "pass@8": 0.7423055698396044,
+            "pass@16": 0.7843204109962838,
+            "pass@32": 0.8204913013870107,
+            "pass@64": 0.8477207957073809,
+            "pass@128": 0.8664688427299704
+          },
+          "15": {
+            "pass@1": 0.36140046296296297,
+            "pass@2": 0.4426244137017595,
+            "pass@4": 0.5132164547139938,
+            "pass@8": 0.5721977881442194,
+            "pass@16": 0.6197088824105395,
+            "pass@32": 0.658967059013711,
+            "pass@64": 0.6955853807708048,
+            "pass@128": 0.7345679012345679
+          },
+          "16": {
+            "pass@1": 0.21766685103244837,
+            "pass@2": 0.2878512385896452,
+            "pass@4": 0.35677422180634516,
+            "pass@8": 0.4192935319550409,
+            "pass@16": 0.4744116086421201,
+            "pass@32": 0.5235822279834264,
+            "pass@64": 0.5713460103429768,
+            "pass@128": 0.6194690265486725
+          },
+          "17": {
+            "pass@1": 0.19837161144578314,
+            "pass@2": 0.26883901254624787,
+            "pass@4": 0.33928608895650064,
+            "pass@8": 0.40176271614166426,
+            "pass@16": 0.4530815948293738,
+            "pass@32": 0.49619312239293656,
+            "pass@64": 0.536260897714176,
+            "pass@128": 0.5692771084337349
+          },
+          "18": {
+            "pass@1": 0.19788306451612903,
+            "pass@2": 0.2630076041402083,
+            "pass@4": 0.3281824866649735,
+            "pass@8": 0.38767325712032175,
+            "pass@16": 0.4352260441352987,
+            "pass@32": 0.47043974442309966,
+            "pass@64": 0.4992493451488665,
+            "pass@128": 0.5258064516129032
+          },
+          "19": {
+            "pass@1": 0.17987632175226587,
+            "pass@2": 0.247976856697671,
+            "pass@4": 0.3147488179347673,
+            "pass@8": 0.37247260884174876,
+            "pass@16": 0.41889090519709676,
+            "pass@32": 0.45572123675521775,
+            "pass@64": 0.4860676102567291,
+            "pass@128": 0.513595166163142
+          },
+          "20": {
+            "pass@1": 0.15899493243243243,
+            "pass@2": 0.2257040505071999,
+            "pass@4": 0.29573212638960666,
+            "pass@8": 0.3606103751183562,
+            "pass@16": 0.41446968279744983,
+            "pass@32": 0.457917316065741,
+            "pass@64": 0.49484972985591286,
+            "pass@128": 0.5285285285285285
+          }
+        }
+      }
+    }
+  }
+}

Qwen3-100M-ep4/composition/checkpoint-85276_ood_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2abdb12f6d038b927e060a53ec87dfd2305d556b14c3e117c4aad58035dc74fc
+size 2568335013

Qwen3-100M-ep4/composition/graphs/checkpoint-85276_ood_generations.jsonl.graph_results.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0faae406ec3d46534b09ef38089b687062513477f9ef33d60d9610f09b4b1fa
+size 443430559

Qwen3-100M-ep4/composition/graphs/composition_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-ep4/composition", "results_dir": "results/Qwen3-100M-ep4/composition/graphs", "per_file": [{"file": "results/Qwen3-100M-ep4/composition/checkpoint-85276_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1026138, "value_mismatch": 6602, "dependency_mismatch": 124380, "answer_mismatch": 5840, "extra_nodes": 8258, "missing_nodes": 8221, "perfect_with_extra": 86, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 338323, "value_mismatch": 2010, "answer_mismatch": 1722, "dependency_mismatch": 40374, "extra_nodes": 2856, "missing_nodes": 2819, "perfect_with_extra": 16}, "teachers_in_school": {"total": 390656, "perfect": 347450, "value_mismatch": 2410, "dependency_mismatch": 42575, "answer_mismatch": 2219, "missing_nodes": 2895, "extra_nodes": 2905, "perfect_with_extra": 8}, "movie_festival_awards": {"total": 382336, "perfect": 340365, "dependency_mismatch": 41431, "value_mismatch": 2182, "answer_mismatch": 1899, "extra_nodes": 2497, "missing_nodes": 2507, "perfect_with_extra": 62}}, "results_path": "results/Qwen3-100M-ep4/composition/graphs/checkpoint-85276_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-ep4/composition/checkpoint-85276_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 258020, "value_mismatch": 824338, "dependency_mismatch": 998879, "answer_mismatch": 708052, "extra_nodes": 67333, "missing_nodes": 765676, "perfect_with_extra": 19, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 270906, "dependency_mismatch": 331115, "answer_mismatch": 231155, "perfect": 91646, "extra_nodes": 25775, "missing_nodes": 243669, "perfect_with_extra": 4}, "movie_festival_awards": {"total": 418816, "perfect": 80786, "value_mismatch": 274419, "dependency_mismatch": 330693, "answer_mismatch": 236421, "missing_nodes": 258294, "extra_nodes": 21013, "perfect_with_extra": 15}, "teachers_in_school": {"total": 430464, "perfect": 85588, "value_mismatch": 279013, "answer_mismatch": 240476, "dependency_mismatch": 337071, "extra_nodes": 20545, "missing_nodes": 263713}}, "results_path": "results/Qwen3-100M-ep4/composition/graphs/checkpoint-85276_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1284158, "value_mismatch": 830940, "dependency_mismatch": 1123259, "answer_mismatch": 713892, "extra_nodes": 75591, "missing_nodes": 773897, "perfect_with_extra": 105, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-ep4/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 429969, "value_mismatch": 272916, "answer_mismatch": 232877, "dependency_mismatch": 371489, "extra_nodes": 28631, "missing_nodes": 246488, "perfect_with_extra": 20}, "teachers_in_school": {"total": 821120, "perfect": 433038, "value_mismatch": 281423, "dependency_mismatch": 379646, "answer_mismatch": 242695, "missing_nodes": 266608, "extra_nodes": 23450, "perfect_with_extra": 8}, "movie_festival_awards": {"total": 801152, "perfect": 421151, "dependency_mismatch": 372124, "value_mismatch": 276601, "answer_mismatch": 238320, "extra_nodes": 23510, "missing_nodes": 260801, "perfect_with_extra": 77}}}}

Qwen3-100M-ep4/composition/graphs/results.json

@@ -0,0 +1,1780 @@
+{
+  "files": {
+    "checkpoint-85276_id_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "crazy_zootopia": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "3": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "4": {
+              "pass@1": 0.99375,
+              "pass@2": 0.996875,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "5": {
+              "pass@1": 0.9555555555555556,
+              "pass@2": 0.9746031746031746,
+              "pass@4": 0.9904761904761905,
+              "pass@8": 0.9968253968253968,
+              "pass@16": 0.9968253968253968,
+              "pass@32": 0.9968253968253968,
+              "pass@64": 0.9968253968253968,
+              "pass@128": 0.9968253968253968
+            },
+            "6": {
+              "pass@1": 0.9640718562874252,
+              "pass@2": 0.9850299401197605,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "7": {
+              "pass@1": 0.8994082840236687,
+              "pass@2": 0.9437869822485208,
+              "pass@4": 0.9644970414201184,
+              "pass@8": 0.985207100591716,
+              "pass@16": 0.9970414201183432,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "8": {
+              "pass@1": 0.8301282051282052,
+              "pass@2": 0.8910256410256411,
+              "pass@4": 0.9391025641025641,
+              "pass@8": 0.9743589743589743,
+              "pass@16": 0.9903846153846154,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "9": {
+              "pass@1": 0.7436619718309859,
+              "pass@2": 0.819718309859155,
+              "pass@4": 0.9014084507042254,
+              "pass@8": 0.9577464788732394,
+              "pass@16": 0.9915492957746479,
+              "pass@32": 0.9971830985915493,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "10": {
+              "pass@1": 0.6666666666666666,
+              "pass@2": 0.7374631268436578,
+              "pass@4": 0.8082595870206489,
+              "pass@8": 0.8790560471976401,
+              "pass@16": 0.9262536873156342,
+              "pass@32": 0.9557522123893806,
+              "pass@64": 0.9646017699115044,
+              "pass@128": 0.9734513274336283
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.99375,
+              "5": 0.9555555555555556,
+              "6": 0.9640718562874252,
+              "7": 0.8994082840236687,
+              "8": 0.8301282051282052,
+              "9": 0.7436619718309859,
+              "10": 0.6666666666666666
+            },
+            "pass@2": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.996875,
+              "5": 0.9746031746031746,
+              "6": 0.9850299401197605,
+              "7": 0.9437869822485208,
+              "8": 0.8910256410256411,
+              "9": 0.819718309859155,
+              "10": 0.7374631268436578
+            },
+            "pass@4": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9904761904761905,
+              "6": 1.0,
+              "7": 0.9644970414201184,
+              "8": 0.9391025641025641,
+              "9": 0.9014084507042254,
+              "10": 0.8082595870206489
+            },
+            "pass@8": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 1.0,
+              "7": 0.985207100591716,
+              "8": 0.9743589743589743,
+              "9": 0.9577464788732394,
+              "10": 0.8790560471976401
+            },
+            "pass@16": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 1.0,
+              "7": 0.9970414201183432,
+              "8": 0.9903846153846154,
+              "9": 0.9915492957746479,
+              "10": 0.9262536873156342
+            },
+            "pass@32": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 0.9971830985915493,
+              "10": 0.9557522123893806
+            },
+            "pass@64": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9646017699115044
+            },
+            "pass@128": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9734513274336283
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.8922661263086795,
+            "pass@2": 0.9257007767646066,
+            "pass@4": 0.9547450185748058,
+            "pass@8": 0.9763593380614657,
+            "pass@16": 0.9888551165146909,
+            "pass@32": 0.994258696386356,
+            "pass@64": 0.9956095913542722,
+            "pass@128": 0.9966227625802094
+          }
+        },
+        "movie_festival_awards": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "3": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "4": {
+              "pass@1": 0.9912280701754386,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "5": {
+              "pass@1": 0.9558823529411765,
+              "pass@2": 0.9705882352941176,
+              "pass@4": 0.9941176470588236,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "6": {
+              "pass@1": 0.9684542586750788,
+              "pass@2": 0.9747634069400631,
+              "pass@4": 0.9873817034700315,
+              "pass@8": 0.9936908517350158,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "7": {
+              "pass@1": 0.879746835443038,
+              "pass@2": 0.9145569620253164,
+              "pass@4": 0.9493670886075949,
+              "pass@8": 0.9841772151898734,
+              "pass@16": 0.9968354430379747,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "8": {
+              "pass@1": 0.8048048048048048,
+              "pass@2": 0.8678678678678678,
+              "pass@4": 0.93993993993994,
+              "pass@8": 0.96996996996997,
+              "pass@16": 0.996996996996997,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "9": {
+              "pass@1": 0.7620578778135049,
+              "pass@2": 0.8392282958199357,
+              "pass@4": 0.9003215434083601,
+              "pass@8": 0.9453376205787781,
+              "pass@16": 0.9678456591639871,
+              "pass@32": 0.9935691318327974,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "10": {
+              "pass@1": 0.608955223880597,
+              "pass@2": 0.6746268656716418,
+              "pass@4": 0.7671641791044777,
+              "pass@8": 0.8567164179104477,
+              "pass@16": 0.9074626865671642,
+              "pass@32": 0.9432835820895522,
+              "pass@64": 0.9641791044776119,
+              "pass@128": 0.9671641791044776
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9912280701754386,
+              "5": 0.9558823529411765,
+              "6": 0.9684542586750788,
+              "7": 0.879746835443038,
+              "8": 0.8048048048048048,
+              "9": 0.7620578778135049,
+              "10": 0.608955223880597
+            },
+            "pass@2": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9705882352941176,
+              "6": 0.9747634069400631,
+              "7": 0.9145569620253164,
+              "8": 0.8678678678678678,
+              "9": 0.8392282958199357,
+              "10": 0.6746268656716418
+            },
+            "pass@4": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9941176470588236,
+              "6": 0.9873817034700315,
+              "7": 0.9493670886075949,
+              "8": 0.93993993993994,
+              "9": 0.9003215434083601,
+              "10": 0.7671641791044777
+            },
+            "pass@8": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 0.9936908517350158,
+              "7": 0.9841772151898734,
+              "8": 0.96996996996997,
+              "9": 0.9453376205787781,
+              "10": 0.8567164179104477
+            },
+            "pass@16": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 0.9968354430379747,
+              "8": 0.996996996996997,
+              "9": 0.9678456591639871,
+              "10": 0.9074626865671642
+            },
+            "pass@32": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 0.9935691318327974,
+              "10": 0.9432835820895522
+            },
+            "pass@64": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9641791044776119
+            },
+            "pass@128": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9671641791044776
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.8875125544024104,
+            "pass@2": 0.916973552058922,
+            "pass@4": 0.9494476062939404,
+            "pass@8": 0.9725477067291597,
+            "pass@16": 0.9856042852360227,
+            "pass@32": 0.9929695346501507,
+            "pass@64": 0.9959825912286575,
+            "pass@128": 0.9963173752929361
+          }
+        },
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "3": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "4": {
+              "pass@1": 0.9822485207100592,
+              "pass@2": 0.985207100591716,
+              "pass@4": 0.9970414201183432,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "5": {
+              "pass@1": 0.9594202898550724,
+              "pass@2": 0.9797101449275363,
+              "pass@4": 0.991304347826087,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "6": {
+              "pass@1": 0.9684813753581661,
+              "pass@2": 0.9770773638968482,
+              "pass@4": 0.9856733524355301,
+              "pass@8": 0.997134670487106,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "7": {
+              "pass@1": 0.8728323699421965,
+              "pass@2": 0.9132947976878613,
+              "pass@4": 0.9566473988439307,
+              "pass@8": 0.9913294797687862,
+              "pass@16": 0.9971098265895953,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "8": {
+              "pass@1": 0.8112676056338028,
+              "pass@2": 0.8816901408450705,
+              "pass@4": 0.9605633802816902,
+              "pass@8": 0.9859154929577465,
+              "pass@16": 0.9971830985915493,
+              "pass@32": 0.9971830985915493,
+              "pass@64": 0.9971830985915493,
+              "pass@128": 0.9971830985915493
+            },
+            "9": {
+              "pass@1": 0.7305389221556886,
+              "pass@2": 0.7724550898203593,
+              "pass@4": 0.8622754491017964,
+              "pass@8": 0.9401197604790419,
+              "pass@16": 0.9670658682634731,
+              "pass@32": 0.9910179640718563,
+              "pass@64": 0.9970059880239521,
+              "pass@128": 1.0
+            },
+            "10": {
+              "pass@1": 0.6717791411042945,
+              "pass@2": 0.7392638036809815,
+              "pass@4": 0.8159509202453987,
+              "pass@8": 0.8926380368098159,
+              "pass@16": 0.9325153374233128,
+              "pass@32": 0.9478527607361963,
+              "pass@64": 0.9631901840490797,
+              "pass@128": 0.9662576687116564
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9822485207100592,
+              "5": 0.9594202898550724,
+              "6": 0.9684813753581661,
+              "7": 0.8728323699421965,
+              "8": 0.8112676056338028,
+              "9": 0.7305389221556886,
+              "10": 0.6717791411042945
+            },
+            "pass@2": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.985207100591716,
+              "5": 0.9797101449275363,
+              "6": 0.9770773638968482,
+              "7": 0.9132947976878613,
+              "8": 0.8816901408450705,
+              "9": 0.7724550898203593,
+              "10": 0.7392638036809815
+            },
+            "pass@4": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9970414201183432,
+              "5": 0.991304347826087,
+              "6": 0.9856733524355301,
+              "7": 0.9566473988439307,
+              "8": 0.9605633802816902,
+              "9": 0.8622754491017964,
+              "10": 0.8159509202453987
+            },
+            "pass@8": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 0.997134670487106,
+              "7": 0.9913294797687862,
+              "8": 0.9859154929577465,
+              "9": 0.9401197604790419,
+              "10": 0.8926380368098159
+            },
+            "pass@16": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 0.9971098265895953,
+              "8": 0.9971830985915493,
+              "9": 0.9670658682634731,
+              "10": 0.9325153374233128
+            },
+            "pass@32": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 0.9971830985915493,
+              "9": 0.9910179640718563,
+              "10": 0.9478527607361963
+            },
+            "pass@64": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 0.9971830985915493,
+              "9": 0.9970059880239521,
+              "10": 0.9631901840490797
+            },
+            "pass@128": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 0.9971830985915493,
+              "9": 1.0,
+              "10": 0.9662576687116564
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.8889252948885976,
+            "pass@2": 0.9171035386631717,
+            "pass@4": 0.9528178243774574,
+            "pass@8": 0.9790301441677588,
+            "pass@16": 0.9885321100917431,
+            "pass@32": 0.9931192660550459,
+            "pass@64": 0.9954128440366973,
+            "pass@128": 0.9960681520314548
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.989,
+            "pass@2": 0.994,
+            "pass@4": 0.999,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.957,
+            "pass@2": 0.975,
+            "pass@4": 0.992,
+            "pass@8": 0.999,
+            "pass@16": 0.999,
+            "pass@32": 0.999,
+            "pass@64": 0.999,
+            "pass@128": 0.999
+          },
+          "6": {
+            "pass@1": 0.967,
+            "pass@2": 0.979,
+            "pass@4": 0.991,
+            "pass@8": 0.997,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.884,
+            "pass@2": 0.924,
+            "pass@4": 0.957,
+            "pass@8": 0.987,
+            "pass@16": 0.997,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.815,
+            "pass@2": 0.88,
+            "pass@4": 0.947,
+            "pass@8": 0.977,
+            "pass@16": 0.995,
+            "pass@32": 0.999,
+            "pass@64": 0.999,
+            "pass@128": 0.999
+          },
+          "9": {
+            "pass@1": 0.745,
+            "pass@2": 0.81,
+            "pass@4": 0.888,
+            "pass@8": 0.948,
+            "pass@16": 0.976,
+            "pass@32": 0.994,
+            "pass@64": 0.999,
+            "pass@128": 1.0
+          },
+          "10": {
+            "pass@1": 0.649,
+            "pass@2": 0.717,
+            "pass@4": 0.797,
+            "pass@8": 0.876,
+            "pass@16": 0.922,
+            "pass@32": 0.949,
+            "pass@64": 0.964,
+            "pass@128": 0.969
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 0.989,
+            "5": 0.957,
+            "6": 0.967,
+            "7": 0.884,
+            "8": 0.815,
+            "9": 0.745,
+            "10": 0.649
+          },
+          "pass@2": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 0.994,
+            "5": 0.975,
+            "6": 0.979,
+            "7": 0.924,
+            "8": 0.88,
+            "9": 0.81,
+            "10": 0.717
+          },
+          "pass@4": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 0.999,
+            "5": 0.992,
+            "6": 0.991,
+            "7": 0.957,
+            "8": 0.947,
+            "9": 0.888,
+            "10": 0.797
+          },
+          "pass@8": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.999,
+            "6": 0.997,
+            "7": 0.987,
+            "8": 0.977,
+            "9": 0.948,
+            "10": 0.876
+          },
+          "pass@16": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.999,
+            "6": 1.0,
+            "7": 0.997,
+            "8": 0.995,
+            "9": 0.976,
+            "10": 0.922
+          },
+          "pass@32": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.999,
+            "6": 1.0,
+            "7": 1.0,
+            "8": 0.999,
+            "9": 0.994,
+            "10": 0.949
+          },
+          "pass@64": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.999,
+            "6": 1.0,
+            "7": 1.0,
+            "8": 0.999,
+            "9": 0.999,
+            "10": 0.964
+          },
+          "pass@128": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.999,
+            "6": 1.0,
+            "7": 1.0,
+            "8": 0.999,
+            "9": 1.0,
+            "10": 0.969
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.8895555555555555,
+          "pass@2": 0.9198888888888889,
+          "pass@4": 0.9523333333333334,
+          "pass@8": 0.976,
+          "pass@16": 0.9876666666666667,
+          "pass@32": 0.9934444444444445,
+          "pass@64": 0.9956666666666667,
+          "pass@128": 0.9963333333333333
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "crazy_zootopia": {
+            "per_op": {
+              "2": 321,
+              "3": 327,
+              "4": 320,
+              "5": 315,
+              "6": 334,
+              "7": 338,
+              "8": 312,
+              "9": 355,
+              "10": 339
+            },
+            "total": 2961
+          },
+          "movie_festival_awards": {
+            "per_op": {
+              "2": 350,
+              "3": 343,
+              "4": 342,
+              "5": 340,
+              "6": 317,
+              "7": 316,
+              "8": 333,
+              "9": 311,
+              "10": 335
+            },
+            "total": 2987
+          },
+          "teachers_in_school": {
+            "per_op": {
+              "2": 329,
+              "3": 330,
+              "4": 338,
+              "5": 345,
+              "6": 349,
+              "7": 346,
+              "8": 355,
+              "9": 334,
+              "10": 326
+            },
+            "total": 3052
+          }
+        },
+        "total": {
+          "per_op": {
+            "2": 1000,
+            "3": 1000,
+            "4": 1000,
+            "5": 1000,
+            "6": 1000,
+            "7": 1000,
+            "8": 1000,
+            "9": 1000,
+            "10": 1000
+          },
+          "total": 9000
+        }
+      }
+    },
+    "checkpoint-85276_ood_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "crazy_zootopia": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.5835694050991501,
+              "pass@2": 0.6458923512747875,
+              "pass@4": 0.71671388101983,
+              "pass@8": 0.8158640226628895,
+              "pass@16": 0.8923512747875354,
+              "pass@32": 0.9206798866855525,
+              "pass@64": 0.943342776203966,
+              "pass@128": 0.9461756373937678
+            },
+            "12": {
+              "pass@1": 0.5790960451977402,
+              "pass@2": 0.6497175141242938,
+              "pass@4": 0.7033898305084746,
+              "pass@8": 0.7598870056497176,
+              "pass@16": 0.8107344632768362,
+              "pass@32": 0.8361581920903954,
+              "pass@64": 0.8502824858757062,
+              "pass@128": 0.867231638418079
+            },
+            "13": {
+              "pass@1": 0.4231974921630094,
+              "pass@2": 0.48589341692789967,
+              "pass@4": 0.5642633228840125,
+              "pass@8": 0.6238244514106583,
+              "pass@16": 0.6865203761755486,
+              "pass@32": 0.7366771159874608,
+              "pass@64": 0.774294670846395,
+              "pass@128": 0.7962382445141066
+            },
+            "14": {
+              "pass@1": 0.26479750778816197,
+              "pass@2": 0.3426791277258567,
+              "pass@4": 0.40186915887850466,
+              "pass@8": 0.4797507788161994,
+              "pass@16": 0.5545171339563862,
+              "pass@32": 0.6074766355140186,
+              "pass@64": 0.6697819314641744,
+              "pass@128": 0.7040498442367601
+            },
+            "15": {
+              "pass@1": 0.15895953757225434,
+              "pass@2": 0.1936416184971098,
+              "pass@4": 0.23699421965317918,
+              "pass@8": 0.2658959537572254,
+              "pass@16": 0.33236994219653176,
+              "pass@32": 0.3901734104046243,
+              "pass@64": 0.45375722543352603,
+              "pass@128": 0.49710982658959535
+            },
+            "16": {
+              "pass@1": 0.04838709677419355,
+              "pass@2": 0.08387096774193549,
+              "pass@4": 0.1064516129032258,
+              "pass@8": 0.12903225806451613,
+              "pass@16": 0.14838709677419354,
+              "pass@32": 0.16451612903225807,
+              "pass@64": 0.19032258064516128,
+              "pass@128": 0.2129032258064516
+            },
+            "17": {
+              "pass@1": 0.031791907514450865,
+              "pass@2": 0.04046242774566474,
+              "pass@4": 0.057803468208092484,
+              "pass@8": 0.06069364161849711,
+              "pass@16": 0.08092485549132948,
+              "pass@32": 0.11560693641618497,
+              "pass@64": 0.1329479768786127,
+              "pass@128": 0.15895953757225434
+            },
+            "18": {
+              "pass@1": 0.008571428571428572,
+              "pass@2": 0.011428571428571429,
+              "pass@4": 0.014285714285714285,
+              "pass@8": 0.02,
+              "pass@16": 0.02857142857142857,
+              "pass@32": 0.037142857142857144,
+              "pass@64": 0.04285714285714286,
+              "pass@128": 0.05142857142857143
+            },
+            "19": {
+              "pass@1": 0.003003003003003003,
+              "pass@2": 0.009009009009009009,
+              "pass@4": 0.009009009009009009,
+              "pass@8": 0.012012012012012012,
+              "pass@16": 0.012012012012012012,
+              "pass@32": 0.015015015015015015,
+              "pass@64": 0.018018018018018018,
+              "pass@128": 0.018018018018018018
+            },
+            "20": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0,
+              "pass@8": 0.003003003003003003,
+              "pass@16": 0.003003003003003003,
+              "pass@32": 0.003003003003003003,
+              "pass@64": 0.003003003003003003,
+              "pass@128": 0.003003003003003003
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.5835694050991501,
+              "12": 0.5790960451977402,
+              "13": 0.4231974921630094,
+              "14": 0.26479750778816197,
+              "15": 0.15895953757225434,
+              "16": 0.04838709677419355,
+              "17": 0.031791907514450865,
+              "18": 0.008571428571428572,
+              "19": 0.003003003003003003,
+              "20": 0.0
+            },
+            "pass@2": {
+              "11": 0.6458923512747875,
+              "12": 0.6497175141242938,
+              "13": 0.48589341692789967,
+              "14": 0.3426791277258567,
+              "15": 0.1936416184971098,
+              "16": 0.08387096774193549,
+              "17": 0.04046242774566474,
+              "18": 0.011428571428571429,
+              "19": 0.009009009009009009,
+              "20": 0.0
+            },
+            "pass@4": {
+              "11": 0.71671388101983,
+              "12": 0.7033898305084746,
+              "13": 0.5642633228840125,
+              "14": 0.40186915887850466,
+              "15": 0.23699421965317918,
+              "16": 0.1064516129032258,
+              "17": 0.057803468208092484,
+              "18": 0.014285714285714285,
+              "19": 0.009009009009009009,
+              "20": 0.0
+            },
+            "pass@8": {
+              "11": 0.8158640226628895,
+              "12": 0.7598870056497176,
+              "13": 0.6238244514106583,
+              "14": 0.4797507788161994,
+              "15": 0.2658959537572254,
+              "16": 0.12903225806451613,
+              "17": 0.06069364161849711,
+              "18": 0.02,
+              "19": 0.012012012012012012,
+              "20": 0.003003003003003003
+            },
+            "pass@16": {
+              "11": 0.8923512747875354,
+              "12": 0.8107344632768362,
+              "13": 0.6865203761755486,
+              "14": 0.5545171339563862,
+              "15": 0.33236994219653176,
+              "16": 0.14838709677419354,
+              "17": 0.08092485549132948,
+              "18": 0.02857142857142857,
+              "19": 0.012012012012012012,
+              "20": 0.003003003003003003
+            },
+            "pass@32": {
+              "11": 0.9206798866855525,
+              "12": 0.8361581920903954,
+              "13": 0.7366771159874608,
+              "14": 0.6074766355140186,
+              "15": 0.3901734104046243,
+              "16": 0.16451612903225807,
+              "17": 0.11560693641618497,
+              "18": 0.037142857142857144,
+              "19": 0.015015015015015015,
+              "20": 0.003003003003003003
+            },
+            "pass@64": {
+              "11": 0.943342776203966,
+              "12": 0.8502824858757062,
+              "13": 0.774294670846395,
+              "14": 0.6697819314641744,
+              "15": 0.45375722543352603,
+              "16": 0.19032258064516128,
+              "17": 0.1329479768786127,
+              "18": 0.04285714285714286,
+              "19": 0.018018018018018018,
+              "20": 0.003003003003003003
+            },
+            "pass@128": {
+              "11": 0.9461756373937678,
+              "12": 0.867231638418079,
+              "13": 0.7962382445141066,
+              "14": 0.7040498442367601,
+              "15": 0.49710982658959535,
+              "16": 0.2129032258064516,
+              "17": 0.15895953757225434,
+              "18": 0.05142857142857143,
+              "19": 0.018018018018018018,
+              "20": 0.003003003003003003
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.21277860326894502,
+            "pass@2": 0.2487369985141159,
+            "pass@4": 0.2835066864784547,
+            "pass@8": 0.31946508172362553,
+            "pass@16": 0.3575037147102526,
+            "pass@32": 0.38514115898959883,
+            "pass@64": 0.4101040118870728,
+            "pass@128": 0.4276374442793462
+          }
+        },
+        "movie_festival_awards": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.5873015873015873,
+              "pass@2": 0.6634920634920635,
+              "pass@4": 0.7555555555555555,
+              "pass@8": 0.8539682539682539,
+              "pass@16": 0.926984126984127,
+              "pass@32": 0.9619047619047619,
+              "pass@64": 0.9714285714285714,
+              "pass@128": 0.9746031746031746
+            },
+            "12": {
+              "pass@1": 0.5902140672782875,
+              "pass@2": 0.6238532110091743,
+              "pass@4": 0.6880733944954128,
+              "pass@8": 0.7706422018348624,
+              "pass@16": 0.8348623853211009,
+              "pass@32": 0.8746177370030581,
+              "pass@64": 0.8868501529051988,
+              "pass@128": 0.8929663608562691
+            },
+            "13": {
+              "pass@1": 0.41358024691358025,
+              "pass@2": 0.4537037037037037,
+              "pass@4": 0.5339506172839507,
+              "pass@8": 0.5925925925925926,
+              "pass@16": 0.6419753086419753,
+              "pass@32": 0.6666666666666666,
+              "pass@64": 0.7006172839506173,
+              "pass@128": 0.7160493827160493
+            },
+            "14": {
+              "pass@1": 0.21364985163204747,
+              "pass@2": 0.2789317507418398,
+              "pass@4": 0.3353115727002967,
+              "pass@8": 0.4035608308605341,
+              "pass@16": 0.4658753709198813,
+              "pass@32": 0.516320474777448,
+              "pass@64": 0.5548961424332344,
+              "pass@128": 0.6320474777448071
+            },
+            "15": {
+              "pass@1": 0.08024691358024691,
+              "pass@2": 0.11728395061728394,
+              "pass@4": 0.15123456790123457,
+              "pass@8": 0.19135802469135801,
+              "pass@16": 0.24074074074074073,
+              "pass@32": 0.2808641975308642,
+              "pass@64": 0.3549382716049383,
+              "pass@128": 0.404320987654321
+            },
+            "16": {
+              "pass@1": 0.038348082595870206,
+              "pass@2": 0.061946902654867256,
+              "pass@4": 0.06489675516224189,
+              "pass@8": 0.07669616519174041,
+              "pass@16": 0.09144542772861357,
+              "pass@32": 0.11504424778761062,
+              "pass@64": 0.13274336283185842,
+              "pass@128": 0.14749262536873156
+            },
+            "17": {
+              "pass@1": 0.02108433734939759,
+              "pass@2": 0.030120481927710843,
+              "pass@4": 0.03614457831325301,
+              "pass@8": 0.045180722891566265,
+              "pass@16": 0.05120481927710843,
+              "pass@32": 0.060240963855421686,
+              "pass@64": 0.06927710843373494,
+              "pass@128": 0.08433734939759036
+            },
+            "18": {
+              "pass@1": 0.012903225806451613,
+              "pass@2": 0.012903225806451613,
+              "pass@4": 0.016129032258064516,
+              "pass@8": 0.025806451612903226,
+              "pass@16": 0.03225806451612903,
+              "pass@32": 0.041935483870967745,
+              "pass@64": 0.04838709677419355,
+              "pass@128": 0.06451612903225806
+            },
+            "19": {
+              "pass@1": 0.0,
+              "pass@2": 0.0,
+              "pass@4": 0.0030211480362537764,
+              "pass@8": 0.006042296072507553,
+              "pass@16": 0.00906344410876133,
+              "pass@32": 0.00906344410876133,
+              "pass@64": 0.00906344410876133,
+              "pass@128": 0.012084592145015106
+            },
+            "20": {
+              "pass@1": 0.003003003003003003,
+              "pass@2": 0.003003003003003003,
+              "pass@4": 0.006006006006006006,
+              "pass@8": 0.009009009009009009,
+              "pass@16": 0.012012012012012012,
+              "pass@32": 0.015015015015015015,
+              "pass@64": 0.015015015015015015,
+              "pass@128": 0.018018018018018018
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.5873015873015873,
+              "12": 0.5902140672782875,
+              "13": 0.41358024691358025,
+              "14": 0.21364985163204747,
+              "15": 0.08024691358024691,
+              "16": 0.038348082595870206,
+              "17": 0.02108433734939759,
+              "18": 0.012903225806451613,
+              "19": 0.0,
+              "20": 0.003003003003003003
+            },
+            "pass@2": {
+              "11": 0.6634920634920635,
+              "12": 0.6238532110091743,
+              "13": 0.4537037037037037,
+              "14": 0.2789317507418398,
+              "15": 0.11728395061728394,
+              "16": 0.061946902654867256,
+              "17": 0.030120481927710843,
+              "18": 0.012903225806451613,
+              "19": 0.0,
+              "20": 0.003003003003003003
+            },
+            "pass@4": {
+              "11": 0.7555555555555555,
+              "12": 0.6880733944954128,
+              "13": 0.5339506172839507,
+              "14": 0.3353115727002967,
+              "15": 0.15123456790123457,
+              "16": 0.06489675516224189,
+              "17": 0.03614457831325301,
+              "18": 0.016129032258064516,
+              "19": 0.0030211480362537764,
+              "20": 0.006006006006006006
+            },
+            "pass@8": {
+              "11": 0.8539682539682539,
+              "12": 0.7706422018348624,
+              "13": 0.5925925925925926,
+              "14": 0.4035608308605341,
+              "15": 0.19135802469135801,
+              "16": 0.07669616519174041,
+              "17": 0.045180722891566265,
+              "18": 0.025806451612903226,
+              "19": 0.006042296072507553,
+              "20": 0.009009009009009009
+            },
+            "pass@16": {
+              "11": 0.926984126984127,
+              "12": 0.8348623853211009,
+              "13": 0.6419753086419753,
+              "14": 0.4658753709198813,
+              "15": 0.24074074074074073,
+              "16": 0.09144542772861357,
+              "17": 0.05120481927710843,
+              "18": 0.03225806451612903,
+              "19": 0.00906344410876133,
+              "20": 0.012012012012012012
+            },
+            "pass@32": {
+              "11": 0.9619047619047619,
+              "12": 0.8746177370030581,
+              "13": 0.6666666666666666,
+              "14": 0.516320474777448,
+              "15": 0.2808641975308642,
+              "16": 0.11504424778761062,
+              "17": 0.060240963855421686,
+              "18": 0.041935483870967745,
+              "19": 0.00906344410876133,
+              "20": 0.015015015015015015
+            },
+            "pass@64": {
+              "11": 0.9714285714285714,
+              "12": 0.8868501529051988,
+              "13": 0.7006172839506173,
+              "14": 0.5548961424332344,
+              "15": 0.3549382716049383,
+              "16": 0.13274336283185842,
+              "17": 0.06927710843373494,
+              "18": 0.04838709677419355,
+              "19": 0.00906344410876133,
+              "20": 0.015015015015015015
+            },
+            "pass@128": {
+              "11": 0.9746031746031746,
+              "12": 0.8929663608562691,
+              "13": 0.7160493827160493,
+              "14": 0.6320474777448071,
+              "15": 0.404320987654321,
+              "16": 0.14749262536873156,
+              "17": 0.08433734939759036,
+              "18": 0.06451612903225806,
+              "19": 0.012084592145015106,
+              "20": 0.018018018018018018
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.19407090464547677,
+            "pass@2": 0.22249388753056235,
+            "pass@4": 0.2567237163814181,
+            "pass@8": 0.29492665036674814,
+            "pass@16": 0.32793398533007334,
+            "pass@32": 0.35146699266503667,
+            "pass@64": 0.37163814180929094,
+            "pass@128": 0.3921149144254279
+          }
+        },
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.6506024096385542,
+              "pass@2": 0.6987951807228916,
+              "pass@4": 0.7801204819277109,
+              "pass@8": 0.8283132530120482,
+              "pass@16": 0.9006024096385542,
+              "pass@32": 0.9337349397590361,
+              "pass@64": 0.9578313253012049,
+              "pass@128": 0.9698795180722891
+            },
+            "12": {
+              "pass@1": 0.5454545454545454,
+              "pass@2": 0.5987460815047022,
+              "pass@4": 0.6614420062695925,
+              "pass@8": 0.7492163009404389,
+              "pass@16": 0.8181818181818182,
+              "pass@32": 0.8432601880877743,
+              "pass@64": 0.8557993730407524,
+              "pass@128": 0.8589341692789969
+            },
+            "13": {
+              "pass@1": 0.40896358543417366,
+              "pass@2": 0.49019607843137253,
+              "pass@4": 0.5686274509803921,
+              "pass@8": 0.6386554621848739,
+              "pass@16": 0.680672268907563,
+              "pass@32": 0.7394957983193278,
+              "pass@64": 0.7787114845938375,
+              "pass@128": 0.8151260504201681
+            },
+            "14": {
+              "pass@1": 0.22807017543859648,
+              "pass@2": 0.26900584795321636,
+              "pass@4": 0.30994152046783624,
+              "pass@8": 0.35964912280701755,
+              "pass@16": 0.40350877192982454,
+              "pass@32": 0.4649122807017544,
+              "pass@64": 0.5321637426900585,
+              "pass@128": 0.5877192982456141
+            },
+            "15": {
+              "pass@1": 0.12121212121212122,
+              "pass@2": 0.1484848484848485,
+              "pass@4": 0.19090909090909092,
+              "pass@8": 0.23636363636363636,
+              "pass@16": 0.27575757575757576,
+              "pass@32": 0.32727272727272727,
+              "pass@64": 0.36666666666666664,
+              "pass@128": 0.4090909090909091
+            },
+            "16": {
+              "pass@1": 0.05128205128205128,
+              "pass@2": 0.07977207977207977,
+              "pass@4": 0.09401709401709402,
+              "pass@8": 0.1111111111111111,
+              "pass@16": 0.1225071225071225,
+              "pass@32": 0.1396011396011396,
+              "pass@64": 0.1623931623931624,
+              "pass@128": 0.19658119658119658
+            },
+            "17": {
+              "pass@1": 0.021739130434782608,
+              "pass@2": 0.037267080745341616,
+              "pass@4": 0.046583850931677016,
+              "pass@8": 0.06521739130434782,
+              "pass@16": 0.06832298136645963,
+              "pass@32": 0.06832298136645963,
+              "pass@64": 0.07763975155279502,
+              "pass@128": 0.09937888198757763
+            },
+            "18": {
+              "pass@1": 0.008823529411764706,
+              "pass@2": 0.008823529411764706,
+              "pass@4": 0.011764705882352941,
+              "pass@8": 0.011764705882352941,
+              "pass@16": 0.023529411764705882,
+              "pass@32": 0.029411764705882353,
+              "pass@64": 0.03235294117647059,
+              "pass@128": 0.041176470588235294
+            },
+            "19": {
+              "pass@1": 0.002976190476190476,
+              "pass@2": 0.002976190476190476,
+              "pass@4": 0.008928571428571428,
+              "pass@8": 0.01488095238095238,
+              "pass@16": 0.01488095238095238,
+              "pass@32": 0.01488095238095238,
+              "pass@64": 0.017857142857142856,
+              "pass@128": 0.023809523809523808
+            },
+            "20": {
+              "pass@1": 0.0,
+              "pass@2": 0.0029940119760479044,
+              "pass@4": 0.0029940119760479044,
+              "pass@8": 0.0029940119760479044,
+              "pass@16": 0.0029940119760479044,
+              "pass@32": 0.0029940119760479044,
+              "pass@64": 0.0029940119760479044,
+              "pass@128": 0.0029940119760479044
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.6506024096385542,
+              "12": 0.5454545454545454,
+              "13": 0.40896358543417366,
+              "14": 0.22807017543859648,
+              "15": 0.12121212121212122,
+              "16": 0.05128205128205128,
+              "17": 0.021739130434782608,
+              "18": 0.008823529411764706,
+              "19": 0.002976190476190476,
+              "20": 0.0
+            },
+            "pass@2": {
+              "11": 0.6987951807228916,
+              "12": 0.5987460815047022,
+              "13": 0.49019607843137253,
+              "14": 0.26900584795321636,
+              "15": 0.1484848484848485,
+              "16": 0.07977207977207977,
+              "17": 0.037267080745341616,
+              "18": 0.008823529411764706,
+              "19": 0.002976190476190476,
+              "20": 0.0029940119760479044
+            },
+            "pass@4": {
+              "11": 0.7801204819277109,
+              "12": 0.6614420062695925,
+              "13": 0.5686274509803921,
+              "14": 0.30994152046783624,
+              "15": 0.19090909090909092,
+              "16": 0.09401709401709402,
+              "17": 0.046583850931677016,
+              "18": 0.011764705882352941,
+              "19": 0.008928571428571428,
+              "20": 0.0029940119760479044
+            },
+            "pass@8": {
+              "11": 0.8283132530120482,
+              "12": 0.7492163009404389,
+              "13": 0.6386554621848739,
+              "14": 0.35964912280701755,
+              "15": 0.23636363636363636,
+              "16": 0.1111111111111111,
+              "17": 0.06521739130434782,
+              "18": 0.011764705882352941,
+              "19": 0.01488095238095238,
+              "20": 0.0029940119760479044
+            },
+            "pass@16": {
+              "11": 0.9006024096385542,
+              "12": 0.8181818181818182,
+              "13": 0.680672268907563,
+              "14": 0.40350877192982454,
+              "15": 0.27575757575757576,
+              "16": 0.1225071225071225,
+              "17": 0.06832298136645963,
+              "18": 0.023529411764705882,
+              "19": 0.01488095238095238,
+              "20": 0.0029940119760479044
+            },
+            "pass@32": {
+              "11": 0.9337349397590361,
+              "12": 0.8432601880877743,
+              "13": 0.7394957983193278,
+              "14": 0.4649122807017544,
+              "15": 0.32727272727272727,
+              "16": 0.1396011396011396,
+              "17": 0.06832298136645963,
+              "18": 0.029411764705882353,
+              "19": 0.01488095238095238,
+              "20": 0.0029940119760479044
+            },
+            "pass@64": {
+              "11": 0.9578313253012049,
+              "12": 0.8557993730407524,
+              "13": 0.7787114845938375,
+              "14": 0.5321637426900585,
+              "15": 0.36666666666666664,
+              "16": 0.1623931623931624,
+              "17": 0.07763975155279502,
+              "18": 0.03235294117647059,
+              "19": 0.017857142857142856,
+              "20": 0.0029940119760479044
+            },
+            "pass@128": {
+              "11": 0.9698795180722891,
+              "12": 0.8589341692789969,
+              "13": 0.8151260504201681,
+              "14": 0.5877192982456141,
+              "15": 0.4090909090909091,
+              "16": 0.19658119658119658,
+              "17": 0.09937888198757763,
+              "18": 0.041176470588235294,
+              "19": 0.023809523809523808,
+              "20": 0.0029940119760479044
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.2030924769550996,
+            "pass@2": 0.23312518584597086,
+            "pass@4": 0.26702349093071664,
+            "pass@8": 0.30121914956883733,
+            "pass@16": 0.3303597977995837,
+            "pass@32": 0.3559322033898305,
+            "pass@64": 0.37823371989295274,
+            "pass@128": 0.4005352363960749
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.607,
+            "pass@2": 0.669,
+            "pass@4": 0.75,
+            "pass@8": 0.832,
+            "pass@16": 0.906,
+            "pass@32": 0.938,
+            "pass@64": 0.957,
+            "pass@128": 0.963
+          },
+          "12": {
+            "pass@1": 0.572,
+            "pass@2": 0.625,
+            "pass@4": 0.685,
+            "pass@8": 0.76,
+            "pass@16": 0.821,
+            "pass@32": 0.851,
+            "pass@64": 0.864,
+            "pass@128": 0.873
+          },
+          "13": {
+            "pass@1": 0.415,
+            "pass@2": 0.477,
+            "pass@4": 0.556,
+            "pass@8": 0.619,
+            "pass@16": 0.67,
+            "pass@32": 0.715,
+            "pass@64": 0.752,
+            "pass@128": 0.777
+          },
+          "14": {
+            "pass@1": 0.235,
+            "pass@2": 0.296,
+            "pass@4": 0.348,
+            "pass@8": 0.413,
+            "pass@16": 0.473,
+            "pass@32": 0.528,
+            "pass@64": 0.584,
+            "pass@128": 0.64
+          },
+          "15": {
+            "pass@1": 0.121,
+            "pass@2": 0.154,
+            "pass@4": 0.194,
+            "pass@8": 0.232,
+            "pass@16": 0.284,
+            "pass@32": 0.334,
+            "pass@64": 0.393,
+            "pass@128": 0.438
+          },
+          "16": {
+            "pass@1": 0.046,
+            "pass@2": 0.075,
+            "pass@4": 0.088,
+            "pass@8": 0.105,
+            "pass@16": 0.12,
+            "pass@32": 0.139,
+            "pass@64": 0.161,
+            "pass@128": 0.185
+          },
+          "17": {
+            "pass@1": 0.025,
+            "pass@2": 0.036,
+            "pass@4": 0.047,
+            "pass@8": 0.057,
+            "pass@16": 0.067,
+            "pass@32": 0.082,
+            "pass@64": 0.094,
+            "pass@128": 0.115
+          },
+          "18": {
+            "pass@1": 0.01,
+            "pass@2": 0.011,
+            "pass@4": 0.014,
+            "pass@8": 0.019,
+            "pass@16": 0.028,
+            "pass@32": 0.036,
+            "pass@64": 0.041,
+            "pass@128": 0.052
+          },
+          "19": {
+            "pass@1": 0.002,
+            "pass@2": 0.004,
+            "pass@4": 0.007,
+            "pass@8": 0.011,
+            "pass@16": 0.012,
+            "pass@32": 0.013,
+            "pass@64": 0.015,
+            "pass@128": 0.018
+          },
+          "20": {
+            "pass@1": 0.001,
+            "pass@2": 0.002,
+            "pass@4": 0.003,
+            "pass@8": 0.005,
+            "pass@16": 0.006,
+            "pass@32": 0.007,
+            "pass@64": 0.007,
+            "pass@128": 0.008
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "11": 0.607,
+            "12": 0.572,
+            "13": 0.415,
+            "14": 0.235,
+            "15": 0.121,
+            "16": 0.046,
+            "17": 0.025,
+            "18": 0.01,
+            "19": 0.002,
+            "20": 0.001
+          },
+          "pass@2": {
+            "11": 0.669,
+            "12": 0.625,
+            "13": 0.477,
+            "14": 0.296,
+            "15": 0.154,
+            "16": 0.075,
+            "17": 0.036,
+            "18": 0.011,
+            "19": 0.004,
+            "20": 0.002
+          },
+          "pass@4": {
+            "11": 0.75,
+            "12": 0.685,
+            "13": 0.556,
+            "14": 0.348,
+            "15": 0.194,
+            "16": 0.088,
+            "17": 0.047,
+            "18": 0.014,
+            "19": 0.007,
+            "20": 0.003
+          },
+          "pass@8": {
+            "11": 0.832,
+            "12": 0.76,
+            "13": 0.619,
+            "14": 0.413,
+            "15": 0.232,
+            "16": 0.105,
+            "17": 0.057,
+            "18": 0.019,
+            "19": 0.011,
+            "20": 0.005
+          },
+          "pass@16": {
+            "11": 0.906,
+            "12": 0.821,
+            "13": 0.67,
+            "14": 0.473,
+            "15": 0.284,
+            "16": 0.12,
+            "17": 0.067,
+            "18": 0.028,
+            "19": 0.012,
+            "20": 0.006
+          },
+          "pass@32": {
+            "11": 0.938,
+            "12": 0.851,
+            "13": 0.715,
+            "14": 0.528,
+            "15": 0.334,
+            "16": 0.139,
+            "17": 0.082,
+            "18": 0.036,
+            "19": 0.013,
+            "20": 0.007
+          },
+          "pass@64": {
+            "11": 0.957,
+            "12": 0.864,
+            "13": 0.752,
+            "14": 0.584,
+            "15": 0.393,
+            "16": 0.161,
+            "17": 0.094,
+            "18": 0.041,
+            "19": 0.015,
+            "20": 0.007
+          },
+          "pass@128": {
+            "11": 0.963,
+            "12": 0.873,
+            "13": 0.777,
+            "14": 0.64,
+            "15": 0.438,
+            "16": 0.185,
+            "17": 0.115,
+            "18": 0.052,
+            "19": 0.018,
+            "20": 0.008
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.2034,
+          "pass@2": 0.2349,
+          "pass@4": 0.2692,
+          "pass@8": 0.3053,
+          "pass@16": 0.3387,
+          "pass@32": 0.3643,
+          "pass@64": 0.3868,
+          "pass@128": 0.4069
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "crazy_zootopia": {
+            "per_op": {
+              "11": 353,
+              "12": 354,
+              "13": 319,
+              "14": 321,
+              "15": 346,
+              "16": 310,
+              "17": 346,
+              "18": 350,
+              "19": 333,
+              "20": 333
+            },
+            "total": 3365
+          },
+          "movie_festival_awards": {
+            "per_op": {
+              "11": 315,
+              "12": 327,
+              "13": 324,
+              "14": 337,
+              "15": 324,
+              "16": 339,
+              "17": 332,
+              "18": 310,
+              "19": 331,
+              "20": 333
+            },
+            "total": 3272
+          },
+          "teachers_in_school": {
+            "per_op": {
+              "11": 332,
+              "12": 319,
+              "13": 357,
+              "14": 342,
+              "15": 330,
+              "16": 351,
+              "17": 322,
+              "18": 340,
+              "19": 336,
+              "20": 334
+            },
+            "total": 3363
+          }
+        },
+        "total": {
+          "per_op": {
+            "11": 1000,
+            "12": 1000,
+            "13": 1000,
+            "14": 1000,
+            "15": 1000,
+            "16": 1000,
+            "17": 1000,
+            "18": 1000,
+            "19": 1000,
+            "20": 1000
+          },
+          "total": 10000
+        }
+      }
+    }
+  }
+}

Qwen3-100M-ep4/composition/summary.csv

@@ -0,0 +1,2 @@
+checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
+checkpoint-85276,0.9949305555555555,0.44683984375,0.7064617598684211,0.0,0.0,0.0,200.0108559027778,327.42891796875,1152000,1280000,2432000

Qwen3-100M-ep4/composition/summary.json

@@ -0,0 +1,645 @@
+[
+  {
+    "checkpoint": "checkpoint-85276",
+    "id_acc": 0.9949305555555555,
+    "ood_acc": 0.44683984375,
+    "total_acc": 0.7064617598684211,
+    "id_avg_loss": 0.0,
+    "ood_avg_loss": 0.0,
+    "total_avg_loss": 0.0,
+    "id_avg_resp_len": 200.0108559027778,
+    "ood_avg_resp_len": 327.42891796875,
+    "count_id": 1152000,
+    "count_ood": 1280000,
+    "count_total": 2432000,
+    "template_metrics": {
+      "crazy_zootopia": {
+        "count": 809728,
+        "correct": 576855,
+        "answer_accuracy": 0.7124058943250079,
+        "avg_response_len": 256.26435790784063,
+        "resp_tokens_sum": 207504426,
+        "pass_at_k": {
+          "pass@1": 0.7124058943250079,
+          "pass@2": 0.7482848133624225,
+          "pass@4": 0.7797965772724061,
+          "pass@8": 0.8060704977403156,
+          "pass@16": 0.8268832725059063,
+          "pass@32": 0.8434735001446694,
+          "pass@64": 0.8574080645300883,
+          "pass@128": 0.8695858362314258
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.964855272861357,
+            "pass@2": 0.9744083600445962,
+            "pass@4": 0.980863755415529,
+            "pass@8": 0.9845045622809016,
+            "pass@16": 0.9864817601267433,
+            "pass@32": 0.988192742885706,
+            "pass@64": 0.9911504413060571,
+            "pass@128": 0.9970501474926253
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9985367063492063,
+            "pass@2": 0.99933172415948,
+            "pass@4": 0.999864562762988,
+            "pass@8": 0.999995076184736,
+            "pass@16": 0.999999996268659,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9997688609467456,
+            "pass@2": 0.9999836200670922,
+            "pass@4": 0.9999999417602385,
+            "pass@8": 0.9999999999999069,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.999824719551282,
+            "pass@2": 0.9999976340096912,
+            "pass@4": 0.9999999996995569,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.997337147887324,
+            "pass@2": 0.9977261700121991,
+            "pass@4": 0.9981745219347582,
+            "pass@8": 0.9988346121771267,
+            "pass@16": 0.9995479293349335,
+            "pass@32": 0.9999462946812021,
+            "pass@64": 0.9999998252073432,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9568431303116147,
+            "pass@2": 0.966704122147621,
+            "pass@4": 0.9737702782725814,
+            "pass@8": 0.9797293556840062,
+            "pass@16": 0.9839171023985908,
+            "pass@32": 0.9864784015256126,
+            "pass@64": 0.9885864447521222,
+            "pass@128": 0.9915014164305949
+          },
+          "12": {
+            "pass@1": 0.8533501059322034,
+            "pass@2": 0.8813041477601309,
+            "pass@4": 0.8969065471688916,
+            "pass@8": 0.9103012428913164,
+            "pass@16": 0.9232330989994582,
+            "pass@32": 0.9356599220900689,
+            "pass@64": 0.9473396438691338,
+            "pass@128": 0.9548022598870056
+          },
+          "13": {
+            "pass@1": 0.760384012539185,
+            "pass@2": 0.8294336077308518,
+            "pass@4": 0.874541486977137,
+            "pass@8": 0.9027689973847379,
+            "pass@16": 0.9198466852847369,
+            "pass@32": 0.9308948171804944,
+            "pass@64": 0.9399058726440269,
+            "pass@128": 0.9498432601880877
+          },
+          "14": {
+            "pass@1": 0.6034852024922118,
+            "pass@2": 0.6929789259695341,
+            "pass@4": 0.7630072742075453,
+            "pass@8": 0.8164932601872855,
+            "pass@16": 0.8526563967709537,
+            "pass@32": 0.8763087593659933,
+            "pass@64": 0.8936422032813724,
+            "pass@128": 0.9065420560747663
+          },
+          "15": {
+            "pass@1": 0.4252619219653179,
+            "pass@2": 0.5253370209594465,
+            "pass@4": 0.6099591160209015,
+            "pass@8": 0.6784789856366449,
+            "pass@16": 0.7314473595873143,
+            "pass@32": 0.7713284427298922,
+            "pass@64": 0.8031345386516523,
+            "pass@128": 0.8294797687861272
+          },
+          "16": {
+            "pass@1": 0.26829637096774195,
+            "pass@2": 0.3590876619253239,
+            "pass@4": 0.44438377874136703,
+            "pass@8": 0.5160205150620084,
+            "pass@16": 0.572072708861716,
+            "pass@32": 0.6173416875891375,
+            "pass@64": 0.6591309169128403,
+            "pass@128": 0.6967741935483871
+          },
+          "17": {
+            "pass@1": 0.210305274566474,
+            "pass@2": 0.2880021534067633,
+            "pass@4": 0.36638274885075767,
+            "pass@8": 0.4354115772958514,
+            "pass@16": 0.4924820483803051,
+            "pass@32": 0.5414044485040482,
+            "pass@64": 0.5843553236837202,
+            "pass@128": 0.6242774566473989
+          },
+          "18": {
+            "pass@1": 0.19066964285714286,
+            "pass@2": 0.2540649606299211,
+            "pass@4": 0.3153792696448659,
+            "pass@8": 0.3744374956178368,
+            "pass@16": 0.4304686242139373,
+            "pass@32": 0.4799023237679476,
+            "pass@64": 0.5191885486106175,
+            "pass@128": 0.5485714285714286
+          },
+          "19": {
+            "pass@1": 0.17203922672672672,
+            "pass@2": 0.2452760339079237,
+            "pass@4": 0.31713847114831356,
+            "pass@8": 0.3807029920118585,
+            "pass@16": 0.43294372077217796,
+            "pass@32": 0.47306262676523364,
+            "pass@64": 0.5035309613007347,
+            "pass@128": 0.5315315315315315
+          },
+          "20": {
+            "pass@1": 0.16786317567567569,
+            "pass@2": 0.23649646201319421,
+            "pass@4": 0.3076818460755471,
+            "pass@8": 0.3678730399252492,
+            "pass@16": 0.41342134802722214,
+            "pass@32": 0.450621229184501,
+            "pass@64": 0.48393762875113866,
+            "pass@128": 0.5135135135135135
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 821120,
+        "correct": 578426,
+        "answer_accuracy": 0.704435405300078,
+        "avg_response_len": 263.05899381332813,
+        "resp_tokens_sum": 216003001,
+        "pass_at_k": {
+          "pass@1": 0.704435405300078,
+          "pass@2": 0.7394285546608842,
+          "pass@4": 0.7702634267559836,
+          "pass@8": 0.7961202574484137,
+          "pass@16": 0.8169219087902136,
+          "pass@32": 0.8337421048859038,
+          "pass@64": 0.8480070379164071,
+          "pass@128": 0.8607950116913484
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9569833205521472,
+            "pass@2": 0.9659142372349164,
+            "pass@4": 0.9719421844700393,
+            "pass@8": 0.9748675318830776,
+            "pass@16": 0.9757861748653367,
+            "pass@32": 0.9762266888126843,
+            "pass@64": 0.9769938650304216,
+            "pass@128": 0.9785276073619632
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9998414855072464,
+            "pass@2": 0.9999967904827115,
+            "pass@4": 0.9999999997282949,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999776146131805,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9948283450704225,
+            "pass@2": 0.9955154153266054,
+            "pass@4": 0.9962789690901313,
+            "pass@8": 0.9971273245981712,
+            "pass@16": 0.997797475275284,
+            "pass@32": 0.9984192527437615,
+            "pass@64": 0.9993013197287653,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9957662799401198,
+            "pass@2": 0.9974428603423076,
+            "pass@4": 0.9988750589372435,
+            "pass@8": 0.9997585789040461,
+            "pass@16": 0.9999876161726882,
+            "pass@32": 0.9999999824517899,
+            "pass@64": 0.9999999999999998,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9643731174698795,
+            "pass@2": 0.9750883455080163,
+            "pass@4": 0.9803072865138847,
+            "pass@8": 0.9823661486047269,
+            "pass@16": 0.9832962407555205,
+            "pass@32": 0.9844320989052803,
+            "pass@64": 0.9860781709515227,
+            "pass@128": 0.9879518072289156
+          },
+          "12": {
+            "pass@1": 0.8863636363636364,
+            "pass@2": 0.9133642287660749,
+            "pass@4": 0.9258904760729354,
+            "pass@8": 0.9354300779654707,
+            "pass@16": 0.945221377305145,
+            "pass@32": 0.9540520042583904,
+            "pass@64": 0.9598380142135106,
+            "pass@128": 0.9623824451410659
+          },
+          "13": {
+            "pass@1": 0.6867997198879552,
+            "pass@2": 0.7681042397273864,
+            "pass@4": 0.824483724986057,
+            "pass@8": 0.8627893286369971,
+            "pass@16": 0.8897924787212615,
+            "pass@32": 0.9118141519741789,
+            "pass@64": 0.9305813041850224,
+            "pass@128": 0.9439775910364145
+          },
+          "14": {
+            "pass@1": 0.5445906432748538,
+            "pass@2": 0.6282847970483952,
+            "pass@4": 0.6919824232497251,
+            "pass@8": 0.7401227434606403,
+            "pass@16": 0.7772650766591268,
+            "pass@32": 0.8067142511689581,
+            "pass@64": 0.8289431092869494,
+            "pass@128": 0.8450292397660819
+          },
+          "15": {
+            "pass@1": 0.3677556818181818,
+            "pass@2": 0.4612525351944643,
+            "pass@4": 0.5395015228210107,
+            "pass@8": 0.6038984673310392,
+            "pass@16": 0.6548475782041733,
+            "pass@32": 0.695211253240757,
+            "pass@64": 0.7307349783554408,
+            "pass@128": 0.7636363636363637
+          },
+          "16": {
+            "pass@1": 0.23595530626780628,
+            "pass@2": 0.30808173217129914,
+            "pass@4": 0.37879288913672104,
+            "pass@8": 0.44199327843430775,
+            "pass@16": 0.49717687261406224,
+            "pass@32": 0.5475518095537767,
+            "pass@64": 0.5956511172414417,
+            "pass@128": 0.6410256410256411
+          },
+          "17": {
+            "pass@1": 0.23090547360248448,
+            "pass@2": 0.31294512948109743,
+            "pass@4": 0.3923752244478753,
+            "pass@8": 0.4620048777355182,
+            "pass@16": 0.5184937076070995,
+            "pass@32": 0.5641996443460395,
+            "pass@64": 0.6058905683329744,
+            "pass@128": 0.6490683229813664
+          },
+          "18": {
+            "pass@1": 0.19181985294117648,
+            "pass@2": 0.2639264416396479,
+            "pass@4": 0.3402037715873749,
+            "pass@8": 0.41292313703410055,
+            "pass@16": 0.4755867515992984,
+            "pass@32": 0.5242663505584598,
+            "pass@64": 0.5607374408176437,
+            "pass@128": 0.5911764705882353
+          },
+          "19": {
+            "pass@1": 0.17740885416666666,
+            "pass@2": 0.24643025578833894,
+            "pass@4": 0.3146818307867766,
+            "pass@8": 0.37225657621985075,
+            "pass@16": 0.4180985090398595,
+            "pass@32": 0.4553241566350863,
+            "pass@64": 0.48681976016382017,
+            "pass@128": 0.5178571428571429
+          },
+          "20": {
+            "pass@1": 0.1394320733532934,
+            "pass@2": 0.2010787725729644,
+            "pass@4": 0.26803790703183067,
+            "pass@8": 0.32950827537338695,
+            "pass@16": 0.37733200724855503,
+            "pass@32": 0.4121459313058152,
+            "pass@64": 0.43980782684006975,
+            "pass@128": 0.46407185628742514
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 801152,
+        "correct": 562834,
+        "answer_accuracy": 0.7025308555679821,
+        "avg_response_len": 282.1113771169516,
+        "resp_tokens_sum": 226014094,
+        "pass_at_k": {
+          "pass@1": 0.7025308555679821,
+          "pass@2": 0.736340083665345,
+          "pass@4": 0.7661264757077474,
+          "pass@8": 0.791289508043559,
+          "pass@16": 0.8116318796222768,
+          "pass@32": 0.8281989841030529,
+          "pass@64": 0.8425176638526883,
+          "pass@128": 0.8555679821057677
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9654617537313432,
+            "pass@2": 0.974182483253026,
+            "pass@4": 0.9814832566638125,
+            "pass@8": 0.9865134381095033,
+            "pass@16": 0.990010567366735,
+            "pass@32": 0.9930920761226464,
+            "pass@64": 0.9962352974533647,
+            "pass@128": 1.0
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9999080882352941,
+            "pass@2": 0.9999992762853173,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9999258306962026,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9954016516516516,
+            "pass@2": 0.9980015694828687,
+            "pass@4": 0.9994780266768457,
+            "pass@8": 0.9999541707109151,
+            "pass@16": 0.9999996450540588,
+            "pass@32": 0.9999999999945218,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.994599075562701,
+            "pass@2": 0.9959007962630074,
+            "pass@4": 0.9973834390074231,
+            "pass@8": 0.9989100635493507,
+            "pass@16": 0.9998018508583297,
+            "pass@32": 0.9999941260810753,
+            "pass@64": 0.9999999994727716,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9669642857142857,
+            "pass@2": 0.977667635295588,
+            "pass@4": 0.9834914713041819,
+            "pass@8": 0.987697919919454,
+            "pass@16": 0.9903756780268099,
+            "pass@32": 0.9920980050763724,
+            "pass@64": 0.9932587543558059,
+            "pass@128": 0.9936507936507937
+          },
+          "12": {
+            "pass@1": 0.8925602064220184,
+            "pass@2": 0.927709778106865,
+            "pass@4": 0.9466711294115757,
+            "pass@8": 0.9594735205095442,
+            "pass@16": 0.968631391961948,
+            "pass@32": 0.9747149407936225,
+            "pass@64": 0.9783565974863963,
+            "pass@128": 0.981651376146789
+          },
+          "13": {
+            "pass@1": 0.6791087962962963,
+            "pass@2": 0.7509379252940609,
+            "pass@4": 0.8003636541960031,
+            "pass@8": 0.836590700158435,
+            "pass@16": 0.8634558161248805,
+            "pass@32": 0.8838359924574606,
+            "pass@64": 0.8998857884792347,
+            "pass@128": 0.9135802469135802
+          },
+          "14": {
+            "pass@1": 0.5279349035608308,
+            "pass@2": 0.6192675354564355,
+            "pass@4": 0.6896984853896232,
+            "pass@8": 0.7423055698396044,
+            "pass@16": 0.7843204109962838,
+            "pass@32": 0.8204913013870107,
+            "pass@64": 0.8477207957073809,
+            "pass@128": 0.8664688427299704
+          },
+          "15": {
+            "pass@1": 0.36140046296296297,
+            "pass@2": 0.4426244137017595,
+            "pass@4": 0.5132164547139938,
+            "pass@8": 0.5721977881442194,
+            "pass@16": 0.6197088824105395,
+            "pass@32": 0.658967059013711,
+            "pass@64": 0.6955853807708048,
+            "pass@128": 0.7345679012345679
+          },
+          "16": {
+            "pass@1": 0.21766685103244837,
+            "pass@2": 0.2878512385896452,
+            "pass@4": 0.35677422180634516,
+            "pass@8": 0.4192935319550409,
+            "pass@16": 0.4744116086421201,
+            "pass@32": 0.5235822279834264,
+            "pass@64": 0.5713460103429768,
+            "pass@128": 0.6194690265486725
+          },
+          "17": {
+            "pass@1": 0.19837161144578314,
+            "pass@2": 0.26883901254624787,
+            "pass@4": 0.33928608895650064,
+            "pass@8": 0.40176271614166426,
+            "pass@16": 0.4530815948293738,
+            "pass@32": 0.49619312239293656,
+            "pass@64": 0.536260897714176,
+            "pass@128": 0.5692771084337349
+          },
+          "18": {
+            "pass@1": 0.19788306451612903,
+            "pass@2": 0.2630076041402083,
+            "pass@4": 0.3281824866649735,
+            "pass@8": 0.38767325712032175,
+            "pass@16": 0.4352260441352987,
+            "pass@32": 0.47043974442309966,
+            "pass@64": 0.4992493451488665,
+            "pass@128": 0.5258064516129032
+          },
+          "19": {
+            "pass@1": 0.17987632175226587,
+            "pass@2": 0.247976856697671,
+            "pass@4": 0.3147488179347673,
+            "pass@8": 0.37247260884174876,
+            "pass@16": 0.41889090519709676,
+            "pass@32": 0.45572123675521775,
+            "pass@64": 0.4860676102567291,
+            "pass@128": 0.513595166163142
+          },
+          "20": {
+            "pass@1": 0.15899493243243243,
+            "pass@2": 0.2257040505071999,
+            "pass@4": 0.29573212638960666,
+            "pass@8": 0.3606103751183562,
+            "pass@16": 0.41446968279744983,
+            "pass@32": 0.457917316065741,
+            "pass@64": 0.49484972985591286,
+            "pass@128": 0.5285285285285285
+          }
+        }
+      }
+    }
+  }
+]

Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_id_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72b5c20ac734fd52011e03baa0922792ae458dfcbd90a7ea26ab05c31d2a740e
+size 1789986458

Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_metrics.json

@@ -0,0 +1,1897 @@
+{
+  "checkpoint": "checkpoint-38646",
+  "id": {
+    "answer_accuracy": 0.9932248263888889,
+    "avg_response_len": 200.3281736111111,
+    "avg_loss": 0.0,
+    "count": 1152000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.9932248263888889,
+      "pass@2": 0.9952680036089231,
+      "pass@4": 0.9965955903532892,
+      "pass@8": 0.9974907332779658,
+      "pass@16": 0.9981196797512945,
+      "pass@32": 0.9985813699315721,
+      "pass@64": 0.9989574187426506,
+      "pass@128": 0.9993333333333333
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.95884375,
+        "pass@2": 0.9682652559055118,
+        "pass@4": 0.9756639486314218,
+        "pass@8": 0.9813791969954038,
+        "pass@16": 0.9855953958960963,
+        "pass@32": 0.9888302276998161,
+        "pass@64": 0.9917058153968907,
+        "pass@128": 0.995
+      },
+      "2": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "3": {
+        "pass@1": 0.999390625,
+        "pass@2": 0.9997413877952757,
+        "pass@4": 0.9999365339332582,
+        "pass@8": 0.9999964696284563,
+        "pass@16": 0.9999999930573713,
+        "pass@32": 0.9999999999999976,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "4": {
+        "pass@1": 0.9999921875,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "5": {
+        "pass@1": 0.9945,
+        "pass@2": 0.9970660679133861,
+        "pass@4": 0.9983636825084363,
+        "pass@8": 0.9990063609804485,
+        "pass@16": 0.9993881074316475,
+        "pass@32": 0.999688172223201,
+        "pass@64": 0.9999404409407284,
+        "pass@128": 1.0
+      },
+      "6": {
+        "pass@1": 0.9991796875,
+        "pass@2": 0.9996582185039371,
+        "pass@4": 0.9999079231346082,
+        "pass@8": 0.9999925579902273,
+        "pass@16": 0.9999999657010183,
+        "pass@32": 0.9999999999998928,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "7": {
+        "pass@1": 0.997015625,
+        "pass@2": 0.9979917568897638,
+        "pass@4": 0.9985803902324709,
+        "pass@8": 0.9990458205281643,
+        "pass@16": 0.9994464582913845,
+        "pass@32": 0.999767693754703,
+        "pass@64": 0.9999711810355609,
+        "pass@128": 1.0
+      },
+      "8": {
+        "pass@1": 0.9955234375,
+        "pass@2": 0.9983201279527558,
+        "pass@4": 0.9996303053055869,
+        "pass@8": 0.999974862902426,
+        "pass@16": 0.9999998996583422,
+        "pass@32": 0.9999999999997113,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "9": {
+        "pass@1": 0.994578125,
+        "pass@2": 0.996369217519685,
+        "pass@4": 0.9972775294338206,
+        "pass@8": 0.9980213304765657,
+        "pass@16": 0.9986472977257898,
+        "pass@32": 0.9989462357068267,
+        "pass@64": 0.9989993313106743,
+        "pass@128": 0.999
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.95884375,
+      "2": 1.0,
+      "3": 0.999390625,
+      "4": 0.9999921875,
+      "5": 0.9945,
+      "6": 0.9991796875,
+      "7": 0.997015625,
+      "8": 0.9955234375,
+      "9": 0.994578125
+    },
+    "per_op_avg_response_len": {
+      "10": 281.7881875,
+      "2": 115.6349609375,
+      "3": 144.9606328125,
+      "4": 159.9038359375,
+      "5": 173.56409375,
+      "6": 199.1092890625,
+      "7": 224.44571875,
+      "8": 243.821984375,
+      "9": 259.724859375
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 281.7881875,
+      "2": 115.6349609375,
+      "3": 144.9606328125,
+      "4": 159.9038359375,
+      "5": 173.56409375,
+      "6": 199.1092890625,
+      "7": 224.44571875,
+      "8": 243.821984375,
+      "9": 259.724859375
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 379008,
+        "correct": 376776,
+        "answer_accuracy": 0.9941109422492401,
+        "avg_response_len": 190.80592494089834,
+        "resp_tokens_sum": 72316972,
+        "pass_at_k": {
+          "pass@1": 0.9941109422492401,
+          "pass@2": 0.9958522471925055,
+          "pass@4": 0.9969521343709948,
+          "pass@8": 0.997695466193822,
+          "pass@16": 0.9982193570151264,
+          "pass@32": 0.9985217552165557,
+          "pass@64": 0.9987341050585805,
+          "pass@128": 0.9989868287740629
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9643482669616519,
+            "pass@2": 0.9724997386941677,
+            "pass@4": 0.9787036025917114,
+            "pass@8": 0.9837184467724707,
+            "pass@16": 0.9876176547055514,
+            "pass@32": 0.990048743242238,
+            "pass@64": 0.9918928791392225,
+            "pass@128": 0.9941002949852508
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9998805428134556,
+            "pass@2": 0.9999977425413568,
+            "pass@4": 0.9999999997133384,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9967757936507936,
+            "pass@2": 0.9988044463192103,
+            "pass@4": 0.9997930303354938,
+            "pass@8": 0.9999940728357846,
+            "pass@16": 0.999999996928095,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9981521332335329,
+            "pass@2": 0.999078737210618,
+            "pass@4": 0.9997273029868273,
+            "pass@8": 0.9999777200738017,
+            "pass@16": 0.9999998973084382,
+            "pass@32": 0.999999999999679,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.998959874260355,
+            "pass@2": 0.9998143607603783,
+            "pass@4": 0.9999912155026479,
+            "pass@8": 0.9999999836752375,
+            "pass@16": 0.9999999999999905,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9960687099358975,
+            "pass@2": 0.9985512252675145,
+            "pass@4": 0.9996649123787412,
+            "pass@8": 0.9999730867256917,
+            "pass@16": 0.9999998586577866,
+            "pass@32": 0.9999999999993815,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9940801056338028,
+            "pass@2": 0.9950447765332151,
+            "pass@4": 0.9956578631896364,
+            "pass@8": 0.996375867480201,
+            "pass@16": 0.9969724240620078,
+            "pass@32": 0.9971729386969103,
+            "pass@64": 0.9971830959162251,
+            "pass@128": 0.9971830985915493
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 390656,
+        "correct": 387835,
+        "answer_accuracy": 0.9927788130733946,
+        "avg_response_len": 198.29017345183487,
+        "resp_tokens_sum": 77463246,
+        "pass_at_k": {
+          "pass@1": 0.9927788130733946,
+          "pass@2": 0.9945622255575279,
+          "pass@4": 0.9957723347122567,
+          "pass@8": 0.9965841163949335,
+          "pass@16": 0.9972021928309309,
+          "pass@32": 0.9978155261463467,
+          "pass@64": 0.9984095685624238,
+          "pass@128": 0.9990170380078637
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9476131134969326,
+            "pass@2": 0.9578519002705184,
+            "pass@4": 0.9660574474778076,
+            "pass@8": 0.9719991236070697,
+            "pass@16": 0.9763583249372905,
+            "pass@32": 0.980656765738378,
+            "pass@64": 0.9852951841811889,
+            "pass@128": 0.99079754601227
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9982717803030303,
+            "pass@2": 0.9992185635886424,
+            "pass@4": 0.9998076788696866,
+            "pass@8": 0.9999893019044129,
+            "pass@16": 0.9999999789617312,
+            "pass@32": 0.9999999999999926,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.9999768860946746,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9955842391304348,
+            "pass@2": 0.9970807657195024,
+            "pass@4": 0.9974464667460045,
+            "pass@8": 0.9977680565376104,
+            "pass@16": 0.9983124881128989,
+            "pass@32": 0.999097406302473,
+            "pass@64": 0.9998273650576287,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9994627507163324,
+            "pass@2": 0.9999027028856349,
+            "pass@4": 0.9999971459513118,
+            "pass@8": 0.9999999985260104,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9984420158959537,
+            "pass@2": 0.9991658037868099,
+            "pass@4": 0.9997657894822396,
+            "pass@8": 0.999983097148883,
+            "pass@16": 0.9999999407613452,
+            "pass@32": 0.9999999999999076,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9972711267605634,
+            "pass@2": 0.9988802401020296,
+            "pass@4": 0.9997421626874106,
+            "pass@8": 0.9999832931204404,
+            "pass@16": 0.9999999422581453,
+            "pass@32": 0.99999999999991,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9965147829341318,
+            "pass@2": 0.9973931320430004,
+            "pass@4": 0.9978456220292824,
+            "pass@8": 0.998468082786387,
+            "pass@16": 0.9992530482905866,
+            "pass@32": 0.999851122735485,
+            "pass@64": 0.9999980007921191,
+            "pass@128": 1.0
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 382336,
+        "correct": 379584,
+        "answer_accuracy": 0.9928021426180114,
+        "avg_response_len": 211.8498859641781,
+        "resp_tokens_sum": 80997838,
+        "pass_at_k": {
+          "pass@1": 0.9928021426180114,
+          "pass@2": 0.9954099819691105,
+          "pass@4": 0.9970833203097687,
+          "pass@8": 0.9982141281769159,
+          "pass@16": 0.9989583224371812,
+          "pass@32": 0.9994229750215191,
+          "pass@64": 0.9997385605466621,
+          "pass@128": 1.0
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9642024253731343,
+            "pass@2": 0.9741138059701494,
+            "pass@4": 0.9819364163434796,
+            "pass@8": 0.9881400872944226,
+            "pass@16": 0.9925379015562921,
+            "pass@32": 0.9955510391342886,
+            "pass@64": 0.997754923360079,
+            "pass@128": 1.0
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9912913602941177,
+            "pass@2": 0.9954405974988421,
+            "pass@4": 0.9979701380158362,
+            "pass@8": 0.9993477897991202,
+            "pass@16": 0.9999126470598457,
+            "pass@32": 0.999998726614258,
+            "pass@64": 0.999999999987784,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999507097791798,
+            "pass@2": 0.999999611888025,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9933742088607594,
+            "pass@2": 0.9947567558307585,
+            "pass@4": 0.9957734057965857,
+            "pass@8": 0.9969989697228501,
+            "pass@16": 0.9982483505948174,
+            "pass@32": 0.9992648536542246,
+            "pass@64": 0.9999088007454459,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9931493993993994,
+            "pass@2": 0.9975064877869996,
+            "pass@4": 0.999478633301468,
+            "pass@8": 0.9999675398986609,
+            "pass@16": 0.9999998926590724,
+            "pass@32": 0.9999999999998089,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9930667202572347,
+            "pass@2": 0.9967814011190724,
+            "pass@4": 0.9985162387257863,
+            "pass@8": 0.9994198002264989,
+            "pass@16": 0.9999085821695214,
+            "pass@32": 0.999998609246854,
+            "pass@64": 0.9999999999866449,
+            "pass@128": 1.0
+          }
+        }
+      }
+    }
+  },
+  "ood": {
+    "answer_accuracy": 0.83598125,
+    "avg_response_len": 378.03591796875,
+    "avg_loss": 0.0,
+    "count": 1280000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.83598125,
+      "pass@2": 0.8769260949802976,
+      "pass@4": 0.9062144027652703,
+      "pass@8": 0.927088376859717,
+      "pass@16": 0.9418586434981504,
+      "pass@32": 0.9525318270976024,
+      "pass@64": 0.9609100985370623,
+      "pass@128": 0.9681
+    },
+    "per_op_pass_at_k": {
+      "11": {
+        "pass@1": 0.970671875,
+        "pass@2": 0.9794998769685037,
+        "pass@4": 0.9857863502999626,
+        "pass@8": 0.9900398112371814,
+        "pass@16": 0.9931495940187157,
+        "pass@32": 0.9953983463354901,
+        "pass@64": 0.9969441662809593,
+        "pass@128": 0.998
+      },
+      "12": {
+        "pass@1": 0.966296875,
+        "pass@2": 0.9809158464566927,
+        "pass@4": 0.9901372777465317,
+        "pass@8": 0.9945470208837859,
+        "pass@16": 0.996475576710408,
+        "pass@32": 0.9975372922449343,
+        "pass@64": 0.9982497123716464,
+        "pass@128": 0.999
+      },
+      "13": {
+        "pass@1": 0.9553671875,
+        "pass@2": 0.973431348425197,
+        "pass@4": 0.9843508790776158,
+        "pass@8": 0.9907141222601963,
+        "pass@16": 0.9945169364443083,
+        "pass@32": 0.99655956862069,
+        "pass@64": 0.9974491256090368,
+        "pass@128": 0.998
+      },
+      "14": {
+        "pass@1": 0.949578125,
+        "pass@2": 0.9686524360236218,
+        "pass@4": 0.9809611437007878,
+        "pass@8": 0.9883708817653398,
+        "pass@16": 0.9925049124837143,
+        "pass@32": 0.9948066901098008,
+        "pass@64": 0.9961112856236247,
+        "pass@128": 0.997
+      },
+      "15": {
+        "pass@1": 0.965375,
+        "pass@2": 0.9803630659448824,
+        "pass@4": 0.988369140607424,
+        "pass@8": 0.9930486759883792,
+        "pass@16": 0.9955475560142018,
+        "pass@32": 0.9967154674849088,
+        "pass@64": 0.997376405398854,
+        "pass@128": 0.998
+      },
+      "16": {
+        "pass@1": 0.887765625,
+        "pass@2": 0.9196377952755908,
+        "pass@4": 0.9421223665166858,
+        "pass@8": 0.9573031508529518,
+        "pass@16": 0.9670828002231159,
+        "pass@32": 0.9741994820696727,
+        "pass@64": 0.9803724490952007,
+        "pass@128": 0.986
+      },
+      "17": {
+        "pass@1": 0.856984375,
+        "pass@2": 0.8984932332677166,
+        "pass@4": 0.924790269216348,
+        "pass@8": 0.9417427006423036,
+        "pass@16": 0.9525526931446574,
+        "pass@32": 0.9599937953920628,
+        "pass@64": 0.9663307158630652,
+        "pass@128": 0.972
+      },
+      "18": {
+        "pass@1": 0.77284375,
+        "pass@2": 0.8381781496062998,
+        "pass@4": 0.8801090434008246,
+        "pass@8": 0.9078199779722668,
+        "pass@16": 0.9261821010586693,
+        "pass@32": 0.9385224656210279,
+        "pass@64": 0.9478627592988191,
+        "pass@128": 0.957
+      },
+      "19": {
+        "pass@1": 0.584125,
+        "pass@2": 0.6814692421259851,
+        "pass@4": 0.7541173871391076,
+        "pass@8": 0.8081890874701764,
+        "pass@16": 0.8491870017860157,
+        "pass@32": 0.8800031510922511,
+        "pass@64": 0.9043406794110973,
+        "pass@128": 0.924
+      },
+      "20": {
+        "pass@1": 0.4508046875,
+        "pass@2": 0.5486199557086607,
+        "pass@4": 0.6314001699475059,
+        "pass@8": 0.6991083395246123,
+        "pass@16": 0.7513872630976118,
+        "pass@32": 0.7915820120051243,
+        "pass@64": 0.8240636864182797,
+        "pass@128": 0.852
+      }
+    },
+    "per_op_accuracy": {
+      "11": 0.970671875,
+      "12": 0.966296875,
+      "13": 0.9553671875,
+      "14": 0.949578125,
+      "15": 0.965375,
+      "16": 0.887765625,
+      "17": 0.856984375,
+      "18": 0.77284375,
+      "19": 0.584125,
+      "20": 0.4508046875
+    },
+    "per_op_avg_response_len": {
+      "11": 301.01784375,
+      "12": 313.407609375,
+      "13": 334.547984375,
+      "14": 361.579859375,
+      "15": 381.650609375,
+      "16": 390.577,
+      "17": 411.0501640625,
+      "18": 426.7855859375,
+      "19": 424.2508671875,
+      "20": 435.49165625
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "11": 301.01784375,
+      "12": 313.407609375,
+      "13": 334.547984375,
+      "14": 361.579859375,
+      "15": 381.650609375,
+      "16": 390.577,
+      "17": 411.0501640625,
+      "18": 426.7855859375,
+      "19": 424.2508671875,
+      "20": 435.49165625
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 430720,
+        "correct": 359051,
+        "answer_accuracy": 0.8336065193164933,
+        "avg_response_len": 360.013558692422,
+        "resp_tokens_sum": 155065040,
+        "pass_at_k": {
+          "pass@1": 0.8336065193164933,
+          "pass@2": 0.8731902853014514,
+          "pass@4": 0.9019876336223236,
+          "pass@8": 0.9227793245050241,
+          "pass@16": 0.9379346127968918,
+          "pass@32": 0.9493515835193701,
+          "pass@64": 0.9586937969666259,
+          "pass@128": 0.9667161961367013
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9711623583569405,
+            "pass@2": 0.9805191998840088,
+            "pass@4": 0.986303336286222,
+            "pass@8": 0.989850641477996,
+            "pass@16": 0.99240868699634,
+            "pass@32": 0.994293234140082,
+            "pass@64": 0.9956686613210934,
+            "pass@128": 0.9971671388101983
+          },
+          "12": {
+            "pass@1": 0.9658368644067796,
+            "pass@2": 0.979869405889942,
+            "pass@4": 0.9899045508612269,
+            "pass@8": 0.9949535165900368,
+            "pass@16": 0.9969359713595375,
+            "pass@32": 0.9982218320963844,
+            "pass@64": 0.9992952648675736,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9498187695924765,
+            "pass@2": 0.9693292893639082,
+            "pass@4": 0.9815207164543299,
+            "pass@8": 0.9885514952344922,
+            "pass@16": 0.993030758542598,
+            "pass@32": 0.9957866897580077,
+            "pass@64": 0.9968002718717178,
+            "pass@128": 0.9968652037617555
+          },
+          "14": {
+            "pass@1": 0.9244548286604362,
+            "pass@2": 0.9464950971373904,
+            "pass@4": 0.962845547343965,
+            "pass@8": 0.9744775229835171,
+            "pass@16": 0.9831241464032126,
+            "pass@32": 0.9892894037314296,
+            "pass@64": 0.9925585252418944,
+            "pass@128": 0.9937694704049844
+          },
+          "15": {
+            "pass@1": 0.9763367052023122,
+            "pass@2": 0.9890843327568163,
+            "pass@4": 0.995195050131017,
+            "pass@8": 0.9987062171803248,
+            "pass@16": 0.9998947926322176,
+            "pass@32": 0.9999995163740522,
+            "pass@64": 0.9999999999997273,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.8670362903225807,
+            "pass@2": 0.9042973710947418,
+            "pass@4": 0.9324377792493678,
+            "pass@8": 0.9511663793058849,
+            "pass@16": 0.9632457765868611,
+            "pass@32": 0.972380137786845,
+            "pass@64": 0.9803184368887754,
+            "pass@128": 0.9870967741935484
+          },
+          "17": {
+            "pass@1": 0.8796739523121387,
+            "pass@2": 0.9130583354194169,
+            "pass@4": 0.9346565702950422,
+            "pass@8": 0.949022076996675,
+            "pass@16": 0.9577801252538675,
+            "pass@32": 0.9642535207391926,
+            "pass@64": 0.9711417638827387,
+            "pass@128": 0.976878612716763
+          },
+          "18": {
+            "pass@1": 0.7535491071428572,
+            "pass@2": 0.8142751687289084,
+            "pass@4": 0.854199288124698,
+            "pass@8": 0.8827058936508151,
+            "pass@16": 0.9034225204407647,
+            "pass@32": 0.9183155907983432,
+            "pass@64": 0.9304605804851537,
+            "pass@128": 0.9428571428571428
+          },
+          "19": {
+            "pass@1": 0.5902073948948949,
+            "pass@2": 0.6823130955758907,
+            "pass@4": 0.7493216109810598,
+            "pass@8": 0.7968110404034479,
+            "pass@16": 0.8325350334716398,
+            "pass@32": 0.859961150550141,
+            "pass@64": 0.8827473329655091,
+            "pass@128": 0.9009009009009009
+          },
+          "20": {
+            "pass@1": 0.4485735735735736,
+            "pass@2": 0.5452467723629141,
+            "pass@4": 0.6280490980294128,
+            "pass@8": 0.6976146762314849,
+            "pass@16": 0.7542370813489142,
+            "pass@32": 0.799241057750153,
+            "pass@64": 0.8368248092112288,
+            "pass@128": 0.8708708708708709
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 430464,
+        "correct": 363264,
+        "answer_accuracy": 0.8438893844781445,
+        "avg_response_len": 373.32513287986916,
+        "resp_tokens_sum": 160703030,
+        "pass_at_k": {
+          "pass@1": 0.8438893844781445,
+          "pass@2": 0.8844832296693336,
+          "pass@4": 0.9135616027200477,
+          "pass@8": 0.9341262715675477,
+          "pass@16": 0.9485360735641319,
+          "pass@32": 0.9589141991748595,
+          "pass@64": 0.9667030192282954,
+          "pass@128": 0.9726434730895034
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9768213478915663,
+            "pass@2": 0.982426507209942,
+            "pass@4": 0.9875867705919291,
+            "pass@8": 0.9916699109297928,
+            "pass@16": 0.9941198985224184,
+            "pass@32": 0.9952745122110707,
+            "pass@64": 0.9962408632343975,
+            "pass@128": 0.9969879518072289
+          },
+          "12": {
+            "pass@1": 0.965492750783699,
+            "pass@2": 0.9797927517093281,
+            "pass@4": 0.9877445484870819,
+            "pass@8": 0.9923678770900078,
+            "pass@16": 0.9955227193860775,
+            "pass@32": 0.9973880317416322,
+            "pass@64": 0.9984300583339356,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9623599439775911,
+            "pass@2": 0.9779697804318578,
+            "pass@4": 0.9872410336838148,
+            "pass@8": 0.9920194344744313,
+            "pass@16": 0.9942491977157041,
+            "pass@32": 0.9950794358980206,
+            "pass@64": 0.9957983155432268,
+            "pass@128": 0.9971988795518207
+          },
+          "14": {
+            "pass@1": 0.9557291666666666,
+            "pass@2": 0.9763930618870013,
+            "pass@4": 0.9898799393496869,
+            "pass@8": 0.9971921128374414,
+            "pass@16": 0.9996866455629769,
+            "pass@32": 0.9999957420781709,
+            "pass@64": 0.9999999999150341,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.9709043560606061,
+            "pass@2": 0.9843362264376043,
+            "pass@4": 0.9913319070911589,
+            "pass@8": 0.9957115636967746,
+            "pass@16": 0.9984277498659098,
+            "pass@32": 0.9997408383719552,
+            "pass@64": 0.9999955357422065,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.9107015669515669,
+            "pass@2": 0.9377124133521775,
+            "pass@4": 0.9564160948898481,
+            "pass@8": 0.9692630161138946,
+            "pass@16": 0.9776624723599159,
+            "pass@32": 0.9832541934776856,
+            "pass@64": 0.9874615955124632,
+            "pass@128": 0.9914529914529915
+          },
+          "17": {
+            "pass@1": 0.8672360248447205,
+            "pass@2": 0.9112839750330124,
+            "pass@4": 0.9387636921937549,
+            "pass@8": 0.9558194693071742,
+            "pass@16": 0.9657225480978907,
+            "pass@32": 0.9716417279156446,
+            "pass@64": 0.9759516660247486,
+            "pass@128": 0.9782608695652174
+          },
+          "18": {
+            "pass@1": 0.7839613970588235,
+            "pass@2": 0.8523347035664656,
+            "pass@4": 0.8947464600013231,
+            "pass@8": 0.9214377697528552,
+            "pass@16": 0.9388117402083257,
+            "pass@32": 0.951005420422579,
+            "pass@64": 0.9607119551856902,
+            "pass@128": 0.9705882352941176
+          },
+          "19": {
+            "pass@1": 0.5933547247023809,
+            "pass@2": 0.6915281830005624,
+            "pass@4": 0.766301561356169,
+            "pass@8": 0.8222817559953494,
+            "pass@16": 0.8649021620408612,
+            "pass@32": 0.8975421565193381,
+            "pass@64": 0.9221817078073432,
+            "pass@128": 0.9404761904761905
+          },
+          "20": {
+            "pass@1": 0.4492889221556886,
+            "pass@2": 0.5484648321467304,
+            "pass@4": 0.6333525578202419,
+            "pass@8": 0.7015890174857582,
+            "pass@16": 0.7546736568184237,
+            "pass@32": 0.796908044139665,
+            "pass@64": 0.829107778372377,
+            "pass@128": 0.8502994011976048
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 418816,
+        "correct": 347741,
+        "answer_accuracy": 0.8302954041870416,
+        "avg_response_len": 401.41232665418704,
+        "resp_tokens_sum": 168117905,
+        "pass_at_k": {
+          "pass@1": 0.8302954041870416,
+          "pass@2": 0.8730007757902922,
+          "pass@4": 0.9030097709554368,
+          "pass@8": 0.9242862745587226,
+          "pass@16": 0.9390310658691144,
+          "pass@32": 0.9492425857604578,
+          "pass@64": 0.9572353621372555,
+          "pass@128": 0.9648533007334963
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.963640873015873,
+            "pass@2": 0.9752730127484065,
+            "pass@4": 0.9833094119187484,
+            "pass@8": 0.9885337281484384,
+            "pass@16": 0.9929572101573484,
+            "pass@32": 0.9967672908887808,
+            "pass@64": 0.9991148007644227,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9675793195718655,
+            "pass@2": 0.9831443087962632,
+            "pass@4": 0.9927234121537561,
+            "pass@8": 0.9962327927192676,
+            "pass@16": 0.9969067136543522,
+            "pass@32": 0.9969418396245668,
+            "pass@64": 0.9969418960244639,
+            "pass@128": 0.9969418960244648
+          },
+          "13": {
+            "pass@1": 0.953125,
+            "pass@2": 0.9724694243948675,
+            "pass@4": 0.9839528441467964,
+            "pass@8": 0.9914051147315478,
+            "pass@16": 0.9962751879157801,
+            "pass@32": 0.9989514073218897,
+            "pass@64": 0.9999069142994649,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9672663204747775,
+            "pass@2": 0.9819023224841703,
+            "pass@4": 0.9891655304029129,
+            "pass@8": 0.9926525053330164,
+            "pass@16": 0.9941520139636356,
+            "pass@32": 0.9947959873034944,
+            "pass@64": 0.9955489586054439,
+            "pass@128": 0.9970326409495549
+          },
+          "15": {
+            "pass@1": 0.9480372299382716,
+            "pass@2": 0.9670028768105379,
+            "pass@4": 0.9780621108704005,
+            "pass@8": 0.9842947803211456,
+            "pass@16": 0.9879716058879758,
+            "pass@32": 0.9901270313479683,
+            "pass@64": 0.9919070327284573,
+            "pass@128": 0.9938271604938271
+          },
+          "16": {
+            "pass@1": 0.882973820058997,
+            "pass@2": 0.9149514842171274,
+            "pass@4": 0.9361787777080962,
+            "pass@8": 0.9505317245196182,
+            "pass@16": 0.95963740909398,
+            "pass@32": 0.9664879570651409,
+            "pass@64": 0.9730817511351202,
+            "pass@128": 0.9793510324483776
+          },
+          "17": {
+            "pass@1": 0.8233951430722891,
+            "pass@2": 0.8709084616023144,
+            "pass@4": 0.900955382553837,
+            "pass@8": 0.920503593025735,
+            "pass@16": 0.9343316545762004,
+            "pass@32": 0.9442573518899536,
+            "pass@64": 0.9519856298182777,
+            "pass@128": 0.9608433734939759
+          },
+          "18": {
+            "pass@1": 0.7824344758064516,
+            "pass@2": 0.8496388430276858,
+            "pass@4": 0.8933080521184847,
+            "pass@8": 0.9212389467048724,
+            "pass@16": 0.9380265394631339,
+            "pass@32": 0.9476456964449388,
+            "pass@64": 0.9534177140834851,
+            "pass@128": 0.9580645161290322
+          },
+          "19": {
+            "pass@1": 0.5686367069486404,
+            "pass@2": 0.6704094012417636,
+            "pass@4": 0.7465739156094909,
+            "pass@8": 0.8053303353516325,
+            "pass@16": 0.8499870368405819,
+            "pass@32": 0.8823623062494177,
+            "pass@64": 0.907953968913336,
+            "pass@128": 0.9305135951661632
+          },
+          "20": {
+            "pass@1": 0.4545561186186186,
+            "pass@2": 0.5521487284528623,
+            "pass@4": 0.6327929909662195,
+            "pass@8": 0.6981138753732286,
+            "pass@16": 0.7452411820752879,
+            "pass@32": 0.7785809400350618,
+            "pass@64": 0.8062433242479462,
+            "pass@128": 0.8348348348348348
+          }
+        }
+      }
+    }
+  },
+  "total": {
+    "answer_accuracy": 0.9104650493421053,
+    "avg_loss": 0.0,
+    "count": 2432000,
+    "validation_metrics": {
+      "id": {},
+      "ood": {}
+    },
+    "pass_at_k": {
+      "pass@1": 0.9104650493421053,
+      "pass@2": 0.9329827885412267,
+      "pass@4": 0.9490265442543343,
+      "pass@8": 0.960436861478886,
+      "pass@16": 0.968508608039099,
+      "pass@32": 0.9743447684400129,
+      "pass@64": 0.9789325133712852,
+      "pass@128": 0.9828947368421053
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.95884375,
+        "pass@2": 0.9682652559055118,
+        "pass@4": 0.9756639486314218,
+        "pass@8": 0.9813791969954038,
+        "pass@16": 0.9855953958960963,
+        "pass@32": 0.9888302276998161,
+        "pass@64": 0.9917058153968907,
+        "pass@128": 0.995
+      },
+      "2": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "3": {
+        "pass@1": 0.999390625,
+        "pass@2": 0.9997413877952757,
+        "pass@4": 0.9999365339332582,
+        "pass@8": 0.9999964696284563,
+        "pass@16": 0.9999999930573713,
+        "pass@32": 0.9999999999999976,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "4": {
+        "pass@1": 0.9999921875,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "5": {
+        "pass@1": 0.9945,
+        "pass@2": 0.9970660679133861,
+        "pass@4": 0.9983636825084363,
+        "pass@8": 0.9990063609804485,
+        "pass@16": 0.9993881074316475,
+        "pass@32": 0.999688172223201,
+        "pass@64": 0.9999404409407284,
+        "pass@128": 1.0
+      },
+      "6": {
+        "pass@1": 0.9991796875,
+        "pass@2": 0.9996582185039371,
+        "pass@4": 0.9999079231346082,
+        "pass@8": 0.9999925579902273,
+        "pass@16": 0.9999999657010183,
+        "pass@32": 0.9999999999998928,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "7": {
+        "pass@1": 0.997015625,
+        "pass@2": 0.9979917568897638,
+        "pass@4": 0.9985803902324709,
+        "pass@8": 0.9990458205281643,
+        "pass@16": 0.9994464582913845,
+        "pass@32": 0.999767693754703,
+        "pass@64": 0.9999711810355609,
+        "pass@128": 1.0
+      },
+      "8": {
+        "pass@1": 0.9955234375,
+        "pass@2": 0.9983201279527558,
+        "pass@4": 0.9996303053055869,
+        "pass@8": 0.999974862902426,
+        "pass@16": 0.9999998996583422,
+        "pass@32": 0.9999999999997113,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "9": {
+        "pass@1": 0.994578125,
+        "pass@2": 0.996369217519685,
+        "pass@4": 0.9972775294338206,
+        "pass@8": 0.9980213304765657,
+        "pass@16": 0.9986472977257898,
+        "pass@32": 0.9989462357068267,
+        "pass@64": 0.9989993313106743,
+        "pass@128": 0.999
+      },
+      "11": {
+        "pass@1": 0.970671875,
+        "pass@2": 0.9794998769685037,
+        "pass@4": 0.9857863502999626,
+        "pass@8": 0.9900398112371814,
+        "pass@16": 0.9931495940187157,
+        "pass@32": 0.9953983463354901,
+        "pass@64": 0.9969441662809593,
+        "pass@128": 0.998
+      },
+      "12": {
+        "pass@1": 0.966296875,
+        "pass@2": 0.9809158464566927,
+        "pass@4": 0.9901372777465317,
+        "pass@8": 0.9945470208837859,
+        "pass@16": 0.996475576710408,
+        "pass@32": 0.9975372922449343,
+        "pass@64": 0.9982497123716464,
+        "pass@128": 0.999
+      },
+      "13": {
+        "pass@1": 0.9553671875,
+        "pass@2": 0.973431348425197,
+        "pass@4": 0.9843508790776158,
+        "pass@8": 0.9907141222601963,
+        "pass@16": 0.9945169364443083,
+        "pass@32": 0.99655956862069,
+        "pass@64": 0.9974491256090368,
+        "pass@128": 0.998
+      },
+      "14": {
+        "pass@1": 0.949578125,
+        "pass@2": 0.9686524360236218,
+        "pass@4": 0.9809611437007878,
+        "pass@8": 0.9883708817653398,
+        "pass@16": 0.9925049124837143,
+        "pass@32": 0.9948066901098008,
+        "pass@64": 0.9961112856236247,
+        "pass@128": 0.997
+      },
+      "15": {
+        "pass@1": 0.965375,
+        "pass@2": 0.9803630659448824,
+        "pass@4": 0.988369140607424,
+        "pass@8": 0.9930486759883792,
+        "pass@16": 0.9955475560142018,
+        "pass@32": 0.9967154674849088,
+        "pass@64": 0.997376405398854,
+        "pass@128": 0.998
+      },
+      "16": {
+        "pass@1": 0.887765625,
+        "pass@2": 0.9196377952755908,
+        "pass@4": 0.9421223665166858,
+        "pass@8": 0.9573031508529518,
+        "pass@16": 0.9670828002231159,
+        "pass@32": 0.9741994820696727,
+        "pass@64": 0.9803724490952007,
+        "pass@128": 0.986
+      },
+      "17": {
+        "pass@1": 0.856984375,
+        "pass@2": 0.8984932332677166,
+        "pass@4": 0.924790269216348,
+        "pass@8": 0.9417427006423036,
+        "pass@16": 0.9525526931446574,
+        "pass@32": 0.9599937953920628,
+        "pass@64": 0.9663307158630652,
+        "pass@128": 0.972
+      },
+      "18": {
+        "pass@1": 0.77284375,
+        "pass@2": 0.8381781496062998,
+        "pass@4": 0.8801090434008246,
+        "pass@8": 0.9078199779722668,
+        "pass@16": 0.9261821010586693,
+        "pass@32": 0.9385224656210279,
+        "pass@64": 0.9478627592988191,
+        "pass@128": 0.957
+      },
+      "19": {
+        "pass@1": 0.584125,
+        "pass@2": 0.6814692421259851,
+        "pass@4": 0.7541173871391076,
+        "pass@8": 0.8081890874701764,
+        "pass@16": 0.8491870017860157,
+        "pass@32": 0.8800031510922511,
+        "pass@64": 0.9043406794110973,
+        "pass@128": 0.924
+      },
+      "20": {
+        "pass@1": 0.4508046875,
+        "pass@2": 0.5486199557086607,
+        "pass@4": 0.6314001699475059,
+        "pass@8": 0.6991083395246123,
+        "pass@16": 0.7513872630976118,
+        "pass@32": 0.7915820120051243,
+        "pass@64": 0.8240636864182797,
+        "pass@128": 0.852
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.95884375,
+      "2": 1.0,
+      "3": 0.999390625,
+      "4": 0.9999921875,
+      "5": 0.9945,
+      "6": 0.9991796875,
+      "7": 0.997015625,
+      "8": 0.9955234375,
+      "9": 0.994578125,
+      "11": 0.970671875,
+      "12": 0.966296875,
+      "13": 0.9553671875,
+      "14": 0.949578125,
+      "15": 0.965375,
+      "16": 0.887765625,
+      "17": 0.856984375,
+      "18": 0.77284375,
+      "19": 0.584125,
+      "20": 0.4508046875
+    },
+    "per_op_avg_response_len": {
+      "10": 281.7881875,
+      "2": 115.6349609375,
+      "3": 144.9606328125,
+      "4": 159.9038359375,
+      "5": 173.56409375,
+      "6": 199.1092890625,
+      "7": 224.44571875,
+      "8": 243.821984375,
+      "9": 259.724859375,
+      "11": 301.01784375,
+      "12": 313.407609375,
+      "13": 334.547984375,
+      "14": 361.579859375,
+      "15": 381.650609375,
+      "16": 390.577,
+      "17": 411.0501640625,
+      "18": 426.7855859375,
+      "19": 424.2508671875,
+      "20": 435.49165625
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 281.7881875,
+      "2": 115.6349609375,
+      "3": 144.9606328125,
+      "4": 159.9038359375,
+      "5": 173.56409375,
+      "6": 199.1092890625,
+      "7": 224.44571875,
+      "8": 243.821984375,
+      "9": 259.724859375,
+      "11": 301.01784375,
+      "12": 313.407609375,
+      "13": 334.547984375,
+      "14": 361.579859375,
+      "15": 381.650609375,
+      "16": 390.577,
+      "17": 411.0501640625,
+      "18": 426.7855859375,
+      "19": 424.2508671875,
+      "20": 435.49165625
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 809728,
+        "correct": 735827,
+        "answer_accuracy": 0.9087335500316156,
+        "avg_response_len": 280.81283097533986,
+        "resp_tokens_sum": 227382012,
+        "pass_at_k": {
+          "pass@1": 0.9087335500316156,
+          "pass@2": 0.9306044600025811,
+          "pass@4": 0.9464375050603238,
+          "pass@8": 0.9578451948086137,
+          "pass@16": 0.9661519899120048,
+          "pass@32": 0.9723665816849368,
+          "pass@64": 0.9774353954902267,
+          "pass@128": 0.9818210559595321
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9643482669616519,
+            "pass@2": 0.9724997386941677,
+            "pass@4": 0.9787036025917114,
+            "pass@8": 0.9837184467724707,
+            "pass@16": 0.9876176547055514,
+            "pass@32": 0.990048743242238,
+            "pass@64": 0.9918928791392225,
+            "pass@128": 0.9941002949852508
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9998805428134556,
+            "pass@2": 0.9999977425413568,
+            "pass@4": 0.9999999997133384,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9967757936507936,
+            "pass@2": 0.9988044463192103,
+            "pass@4": 0.9997930303354938,
+            "pass@8": 0.9999940728357846,
+            "pass@16": 0.999999996928095,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9981521332335329,
+            "pass@2": 0.999078737210618,
+            "pass@4": 0.9997273029868273,
+            "pass@8": 0.9999777200738017,
+            "pass@16": 0.9999998973084382,
+            "pass@32": 0.999999999999679,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.998959874260355,
+            "pass@2": 0.9998143607603783,
+            "pass@4": 0.9999912155026479,
+            "pass@8": 0.9999999836752375,
+            "pass@16": 0.9999999999999905,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9960687099358975,
+            "pass@2": 0.9985512252675145,
+            "pass@4": 0.9996649123787412,
+            "pass@8": 0.9999730867256917,
+            "pass@16": 0.9999998586577866,
+            "pass@32": 0.9999999999993815,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9940801056338028,
+            "pass@2": 0.9950447765332151,
+            "pass@4": 0.9956578631896364,
+            "pass@8": 0.996375867480201,
+            "pass@16": 0.9969724240620078,
+            "pass@32": 0.9971729386969103,
+            "pass@64": 0.9971830959162251,
+            "pass@128": 0.9971830985915493
+          },
+          "11": {
+            "pass@1": 0.9711623583569405,
+            "pass@2": 0.9805191998840088,
+            "pass@4": 0.986303336286222,
+            "pass@8": 0.989850641477996,
+            "pass@16": 0.99240868699634,
+            "pass@32": 0.994293234140082,
+            "pass@64": 0.9956686613210934,
+            "pass@128": 0.9971671388101983
+          },
+          "12": {
+            "pass@1": 0.9658368644067796,
+            "pass@2": 0.979869405889942,
+            "pass@4": 0.9899045508612269,
+            "pass@8": 0.9949535165900368,
+            "pass@16": 0.9969359713595375,
+            "pass@32": 0.9982218320963844,
+            "pass@64": 0.9992952648675736,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9498187695924765,
+            "pass@2": 0.9693292893639082,
+            "pass@4": 0.9815207164543299,
+            "pass@8": 0.9885514952344922,
+            "pass@16": 0.993030758542598,
+            "pass@32": 0.9957866897580077,
+            "pass@64": 0.9968002718717178,
+            "pass@128": 0.9968652037617555
+          },
+          "14": {
+            "pass@1": 0.9244548286604362,
+            "pass@2": 0.9464950971373904,
+            "pass@4": 0.962845547343965,
+            "pass@8": 0.9744775229835171,
+            "pass@16": 0.9831241464032126,
+            "pass@32": 0.9892894037314296,
+            "pass@64": 0.9925585252418944,
+            "pass@128": 0.9937694704049844
+          },
+          "15": {
+            "pass@1": 0.9763367052023122,
+            "pass@2": 0.9890843327568163,
+            "pass@4": 0.995195050131017,
+            "pass@8": 0.9987062171803248,
+            "pass@16": 0.9998947926322176,
+            "pass@32": 0.9999995163740522,
+            "pass@64": 0.9999999999997273,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.8670362903225807,
+            "pass@2": 0.9042973710947418,
+            "pass@4": 0.9324377792493678,
+            "pass@8": 0.9511663793058849,
+            "pass@16": 0.9632457765868611,
+            "pass@32": 0.972380137786845,
+            "pass@64": 0.9803184368887754,
+            "pass@128": 0.9870967741935484
+          },
+          "17": {
+            "pass@1": 0.8796739523121387,
+            "pass@2": 0.9130583354194169,
+            "pass@4": 0.9346565702950422,
+            "pass@8": 0.949022076996675,
+            "pass@16": 0.9577801252538675,
+            "pass@32": 0.9642535207391926,
+            "pass@64": 0.9711417638827387,
+            "pass@128": 0.976878612716763
+          },
+          "18": {
+            "pass@1": 0.7535491071428572,
+            "pass@2": 0.8142751687289084,
+            "pass@4": 0.854199288124698,
+            "pass@8": 0.8827058936508151,
+            "pass@16": 0.9034225204407647,
+            "pass@32": 0.9183155907983432,
+            "pass@64": 0.9304605804851537,
+            "pass@128": 0.9428571428571428
+          },
+          "19": {
+            "pass@1": 0.5902073948948949,
+            "pass@2": 0.6823130955758907,
+            "pass@4": 0.7493216109810598,
+            "pass@8": 0.7968110404034479,
+            "pass@16": 0.8325350334716398,
+            "pass@32": 0.859961150550141,
+            "pass@64": 0.8827473329655091,
+            "pass@128": 0.9009009009009009
+          },
+          "20": {
+            "pass@1": 0.4485735735735736,
+            "pass@2": 0.5452467723629141,
+            "pass@4": 0.6280490980294128,
+            "pass@8": 0.6976146762314849,
+            "pass@16": 0.7542370813489142,
+            "pass@32": 0.799241057750153,
+            "pass@64": 0.8368248092112288,
+            "pass@128": 0.8708708708708709
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 821120,
+        "correct": 751099,
+        "answer_accuracy": 0.9147250097427904,
+        "avg_response_len": 290.0505114964926,
+        "resp_tokens_sum": 238166276,
+        "pass_at_k": {
+          "pass@1": 0.9147250097427904,
+          "pass@2": 0.9368544058892373,
+          "pass@4": 0.9526741754464987,
+          "pass@8": 0.9638412119281327,
+          "pass@16": 0.9716894634319847,
+          "pass@32": 0.9774218920691676,
+          "pass@64": 0.9817877251624777,
+          "pass@128": 0.9851909586905689
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9476131134969326,
+            "pass@2": 0.9578519002705184,
+            "pass@4": 0.9660574474778076,
+            "pass@8": 0.9719991236070697,
+            "pass@16": 0.9763583249372905,
+            "pass@32": 0.980656765738378,
+            "pass@64": 0.9852951841811889,
+            "pass@128": 0.99079754601227
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9982717803030303,
+            "pass@2": 0.9992185635886424,
+            "pass@4": 0.9998076788696866,
+            "pass@8": 0.9999893019044129,
+            "pass@16": 0.9999999789617312,
+            "pass@32": 0.9999999999999926,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.9999768860946746,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9955842391304348,
+            "pass@2": 0.9970807657195024,
+            "pass@4": 0.9974464667460045,
+            "pass@8": 0.9977680565376104,
+            "pass@16": 0.9983124881128989,
+            "pass@32": 0.999097406302473,
+            "pass@64": 0.9998273650576287,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9994627507163324,
+            "pass@2": 0.9999027028856349,
+            "pass@4": 0.9999971459513118,
+            "pass@8": 0.9999999985260104,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9984420158959537,
+            "pass@2": 0.9991658037868099,
+            "pass@4": 0.9997657894822396,
+            "pass@8": 0.999983097148883,
+            "pass@16": 0.9999999407613452,
+            "pass@32": 0.9999999999999076,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9972711267605634,
+            "pass@2": 0.9988802401020296,
+            "pass@4": 0.9997421626874106,
+            "pass@8": 0.9999832931204404,
+            "pass@16": 0.9999999422581453,
+            "pass@32": 0.99999999999991,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9965147829341318,
+            "pass@2": 0.9973931320430004,
+            "pass@4": 0.9978456220292824,
+            "pass@8": 0.998468082786387,
+            "pass@16": 0.9992530482905866,
+            "pass@32": 0.999851122735485,
+            "pass@64": 0.9999980007921191,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9768213478915663,
+            "pass@2": 0.982426507209942,
+            "pass@4": 0.9875867705919291,
+            "pass@8": 0.9916699109297928,
+            "pass@16": 0.9941198985224184,
+            "pass@32": 0.9952745122110707,
+            "pass@64": 0.9962408632343975,
+            "pass@128": 0.9969879518072289
+          },
+          "12": {
+            "pass@1": 0.965492750783699,
+            "pass@2": 0.9797927517093281,
+            "pass@4": 0.9877445484870819,
+            "pass@8": 0.9923678770900078,
+            "pass@16": 0.9955227193860775,
+            "pass@32": 0.9973880317416322,
+            "pass@64": 0.9984300583339356,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9623599439775911,
+            "pass@2": 0.9779697804318578,
+            "pass@4": 0.9872410336838148,
+            "pass@8": 0.9920194344744313,
+            "pass@16": 0.9942491977157041,
+            "pass@32": 0.9950794358980206,
+            "pass@64": 0.9957983155432268,
+            "pass@128": 0.9971988795518207
+          },
+          "14": {
+            "pass@1": 0.9557291666666666,
+            "pass@2": 0.9763930618870013,
+            "pass@4": 0.9898799393496869,
+            "pass@8": 0.9971921128374414,
+            "pass@16": 0.9996866455629769,
+            "pass@32": 0.9999957420781709,
+            "pass@64": 0.9999999999150341,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.9709043560606061,
+            "pass@2": 0.9843362264376043,
+            "pass@4": 0.9913319070911589,
+            "pass@8": 0.9957115636967746,
+            "pass@16": 0.9984277498659098,
+            "pass@32": 0.9997408383719552,
+            "pass@64": 0.9999955357422065,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.9107015669515669,
+            "pass@2": 0.9377124133521775,
+            "pass@4": 0.9564160948898481,
+            "pass@8": 0.9692630161138946,
+            "pass@16": 0.9776624723599159,
+            "pass@32": 0.9832541934776856,
+            "pass@64": 0.9874615955124632,
+            "pass@128": 0.9914529914529915
+          },
+          "17": {
+            "pass@1": 0.8672360248447205,
+            "pass@2": 0.9112839750330124,
+            "pass@4": 0.9387636921937549,
+            "pass@8": 0.9558194693071742,
+            "pass@16": 0.9657225480978907,
+            "pass@32": 0.9716417279156446,
+            "pass@64": 0.9759516660247486,
+            "pass@128": 0.9782608695652174
+          },
+          "18": {
+            "pass@1": 0.7839613970588235,
+            "pass@2": 0.8523347035664656,
+            "pass@4": 0.8947464600013231,
+            "pass@8": 0.9214377697528552,
+            "pass@16": 0.9388117402083257,
+            "pass@32": 0.951005420422579,
+            "pass@64": 0.9607119551856902,
+            "pass@128": 0.9705882352941176
+          },
+          "19": {
+            "pass@1": 0.5933547247023809,
+            "pass@2": 0.6915281830005624,
+            "pass@4": 0.766301561356169,
+            "pass@8": 0.8222817559953494,
+            "pass@16": 0.8649021620408612,
+            "pass@32": 0.8975421565193381,
+            "pass@64": 0.9221817078073432,
+            "pass@128": 0.9404761904761905
+          },
+          "20": {
+            "pass@1": 0.4492889221556886,
+            "pass@2": 0.5484648321467304,
+            "pass@4": 0.6333525578202419,
+            "pass@8": 0.7015890174857582,
+            "pass@16": 0.7546736568184237,
+            "pass@32": 0.796908044139665,
+            "pass@64": 0.829107778372377,
+            "pass@128": 0.8502994011976048
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 801152,
+        "correct": 727325,
+        "answer_accuracy": 0.9078489475155775,
+        "avg_response_len": 310.94691519212336,
+        "resp_tokens_sum": 249115743,
+        "pass_at_k": {
+          "pass@1": 0.9078489475155775,
+          "pass@2": 0.9314184621389209,
+          "pass@4": 0.9479047528888704,
+          "pass@8": 0.959567070014469,
+          "pass@16": 0.9676303174059113,
+          "pass@32": 0.9731903126693562,
+          "pass@64": 0.9775192818766577,
+          "pass@128": 0.9816264579006231
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9642024253731343,
+            "pass@2": 0.9741138059701494,
+            "pass@4": 0.9819364163434796,
+            "pass@8": 0.9881400872944226,
+            "pass@16": 0.9925379015562921,
+            "pass@32": 0.9955510391342886,
+            "pass@64": 0.997754923360079,
+            "pass@128": 1.0
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9912913602941177,
+            "pass@2": 0.9954405974988421,
+            "pass@4": 0.9979701380158362,
+            "pass@8": 0.9993477897991202,
+            "pass@16": 0.9999126470598457,
+            "pass@32": 0.999998726614258,
+            "pass@64": 0.999999999987784,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999507097791798,
+            "pass@2": 0.999999611888025,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9933742088607594,
+            "pass@2": 0.9947567558307585,
+            "pass@4": 0.9957734057965857,
+            "pass@8": 0.9969989697228501,
+            "pass@16": 0.9982483505948174,
+            "pass@32": 0.9992648536542246,
+            "pass@64": 0.9999088007454459,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9931493993993994,
+            "pass@2": 0.9975064877869996,
+            "pass@4": 0.999478633301468,
+            "pass@8": 0.9999675398986609,
+            "pass@16": 0.9999998926590724,
+            "pass@32": 0.9999999999998089,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9930667202572347,
+            "pass@2": 0.9967814011190724,
+            "pass@4": 0.9985162387257863,
+            "pass@8": 0.9994198002264989,
+            "pass@16": 0.9999085821695214,
+            "pass@32": 0.999998609246854,
+            "pass@64": 0.9999999999866449,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.963640873015873,
+            "pass@2": 0.9752730127484065,
+            "pass@4": 0.9833094119187484,
+            "pass@8": 0.9885337281484384,
+            "pass@16": 0.9929572101573484,
+            "pass@32": 0.9967672908887808,
+            "pass@64": 0.9991148007644227,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9675793195718655,
+            "pass@2": 0.9831443087962632,
+            "pass@4": 0.9927234121537561,
+            "pass@8": 0.9962327927192676,
+            "pass@16": 0.9969067136543522,
+            "pass@32": 0.9969418396245668,
+            "pass@64": 0.9969418960244639,
+            "pass@128": 0.9969418960244648
+          },
+          "13": {
+            "pass@1": 0.953125,
+            "pass@2": 0.9724694243948675,
+            "pass@4": 0.9839528441467964,
+            "pass@8": 0.9914051147315478,
+            "pass@16": 0.9962751879157801,
+            "pass@32": 0.9989514073218897,
+            "pass@64": 0.9999069142994649,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9672663204747775,
+            "pass@2": 0.9819023224841703,
+            "pass@4": 0.9891655304029129,
+            "pass@8": 0.9926525053330164,
+            "pass@16": 0.9941520139636356,
+            "pass@32": 0.9947959873034944,
+            "pass@64": 0.9955489586054439,
+            "pass@128": 0.9970326409495549
+          },
+          "15": {
+            "pass@1": 0.9480372299382716,
+            "pass@2": 0.9670028768105379,
+            "pass@4": 0.9780621108704005,
+            "pass@8": 0.9842947803211456,
+            "pass@16": 0.9879716058879758,
+            "pass@32": 0.9901270313479683,
+            "pass@64": 0.9919070327284573,
+            "pass@128": 0.9938271604938271
+          },
+          "16": {
+            "pass@1": 0.882973820058997,
+            "pass@2": 0.9149514842171274,
+            "pass@4": 0.9361787777080962,
+            "pass@8": 0.9505317245196182,
+            "pass@16": 0.95963740909398,
+            "pass@32": 0.9664879570651409,
+            "pass@64": 0.9730817511351202,
+            "pass@128": 0.9793510324483776
+          },
+          "17": {
+            "pass@1": 0.8233951430722891,
+            "pass@2": 0.8709084616023144,
+            "pass@4": 0.900955382553837,
+            "pass@8": 0.920503593025735,
+            "pass@16": 0.9343316545762004,
+            "pass@32": 0.9442573518899536,
+            "pass@64": 0.9519856298182777,
+            "pass@128": 0.9608433734939759
+          },
+          "18": {
+            "pass@1": 0.7824344758064516,
+            "pass@2": 0.8496388430276858,
+            "pass@4": 0.8933080521184847,
+            "pass@8": 0.9212389467048724,
+            "pass@16": 0.9380265394631339,
+            "pass@32": 0.9476456964449388,
+            "pass@64": 0.9534177140834851,
+            "pass@128": 0.9580645161290322
+          },
+          "19": {
+            "pass@1": 0.5686367069486404,
+            "pass@2": 0.6704094012417636,
+            "pass@4": 0.7465739156094909,
+            "pass@8": 0.8053303353516325,
+            "pass@16": 0.8499870368405819,
+            "pass@32": 0.8823623062494177,
+            "pass@64": 0.907953968913336,
+            "pass@128": 0.9305135951661632
+          },
+          "20": {
+            "pass@1": 0.4545561186186186,
+            "pass@2": 0.5521487284528623,
+            "pass@4": 0.6327929909662195,
+            "pass@8": 0.6981138753732286,
+            "pass@16": 0.7452411820752879,
+            "pass@32": 0.7785809400350618,
+            "pass@64": 0.8062433242479462,
+            "pass@128": 0.8348348348348348
+          }
+        }
+      }
+    }
+  }
+}

Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_ood_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0856a41fc6d7287904e654c1ed69815ee5a8f798e27f64f141ecc468cdecafbb
+size 2712047402

Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed9a1aa48fa3d36bc185d3cb2f6d68fc7570f68245d5df5e58c01a6513a1f052
+size 373191713

Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5be0f75384b390f6e7f5e8b423d9aac35197e5a6a4420a6907c1ef23577736c4
+size 423962866

Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/composition_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-midtrain-diff_ar-block16/composition", "results_dir": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs", "per_file": [{"file": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024126, "value_mismatch": 8591, "dependency_mismatch": 126456, "answer_mismatch": 7805, "extra_nodes": 12243, "missing_nodes": 9153, "perfect_with_extra": 1350, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337616, "dependency_mismatch": 40996, "extra_nodes": 3880, "perfect_with_extra": 438, "value_mismatch": 2510, "answer_mismatch": 2232, "missing_nodes": 2903}, "teachers_in_school": {"total": 390656, "perfect": 347014, "dependency_mismatch": 43146, "extra_nodes": 4142, "perfect_with_extra": 377, "value_mismatch": 2961, "answer_mismatch": 2821, "missing_nodes": 3349}, "movie_festival_awards": {"total": 382336, "perfect": 339496, "dependency_mismatch": 42314, "value_mismatch": 3120, "answer_mismatch": 2752, "extra_nodes": 4221, "missing_nodes": 2901, "perfect_with_extra": 535}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/checkpoint-38646_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 619464, "value_mismatch": 264879, "dependency_mismatch": 640281, "answer_mismatch": 209944, "extra_nodes": 42043, "missing_nodes": 154289, "perfect_with_extra": 301, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 208964, "value_mismatch": 88980, "dependency_mismatch": 214494, "answer_mismatch": 71669, "extra_nodes": 15885, "missing_nodes": 55017, "perfect_with_extra": 70}, "movie_festival_awards": {"total": 418816, "perfect": 201602, "value_mismatch": 90032, "dependency_mismatch": 210582, "answer_mismatch": 71075, "missing_nodes": 48991, "extra_nodes": 13389, "perfect_with_extra": 47}, "teachers_in_school": {"total": 430464, "perfect": 208898, "extra_nodes": 12769, "perfect_with_extra": 184, "dependency_mismatch": 215205, "missing_nodes": 50281, "value_mismatch": 85867, "answer_mismatch": 67200}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1643590, "value_mismatch": 273470, "dependency_mismatch": 766737, "answer_mismatch": 217749, "extra_nodes": 54286, "missing_nodes": 163442, "perfect_with_extra": 1651, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 546580, "dependency_mismatch": 255490, "extra_nodes": 19765, "perfect_with_extra": 508, "value_mismatch": 91490, "answer_mismatch": 73901, "missing_nodes": 57920}, "teachers_in_school": {"total": 821120, "perfect": 555912, "dependency_mismatch": 258351, "extra_nodes": 16911, "perfect_with_extra": 561, "value_mismatch": 88828, "answer_mismatch": 70021, "missing_nodes": 53630}, "movie_festival_awards": {"total": 801152, "perfect": 541098, "dependency_mismatch": 252896, "value_mismatch": 93152, "answer_mismatch": 73827, "extra_nodes": 17610, "missing_nodes": 51892, "perfect_with_extra": 582}}}}

Qwen3-100M-midtrain-diff_ar-block16/composition/graphs/results.json

@@ -0,0 +1,1780 @@
+{
+  "files": {
+    "checkpoint-38646_id_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "crazy_zootopia": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "3": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "4": {
+              "pass@1": 0.996875,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "5": {
+              "pass@1": 0.9619047619047619,
+              "pass@2": 0.9809523809523809,
+              "pass@4": 0.9968253968253968,
+              "pass@8": 0.9968253968253968,
+              "pass@16": 0.9968253968253968,
+              "pass@32": 0.9968253968253968,
+              "pass@64": 0.9968253968253968,
+              "pass@128": 0.9968253968253968
+            },
+            "6": {
+              "pass@1": 0.9580838323353293,
+              "pass@2": 0.9700598802395209,
+              "pass@4": 0.9850299401197605,
+              "pass@8": 0.9910179640718563,
+              "pass@16": 0.9970059880239521,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "7": {
+              "pass@1": 0.8875739644970414,
+              "pass@2": 0.9556213017751479,
+              "pass@4": 0.9733727810650887,
+              "pass@8": 0.9970414201183432,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "8": {
+              "pass@1": 0.8076923076923077,
+              "pass@2": 0.8782051282051282,
+              "pass@4": 0.9358974358974359,
+              "pass@8": 0.9775641025641025,
+              "pass@16": 0.9871794871794872,
+              "pass@32": 0.9967948717948718,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "9": {
+              "pass@1": 0.7098591549295775,
+              "pass@2": 0.7887323943661971,
+              "pass@4": 0.8901408450704226,
+              "pass@8": 0.9323943661971831,
+              "pass@16": 0.9774647887323944,
+              "pass@32": 0.9971830985915493,
+              "pass@64": 0.9971830985915493,
+              "pass@128": 0.9971830985915493
+            },
+            "10": {
+              "pass@1": 0.6430678466076696,
+              "pass@2": 0.696165191740413,
+              "pass@4": 0.7905604719764012,
+              "pass@8": 0.8790560471976401,
+              "pass@16": 0.9351032448377581,
+              "pass@32": 0.9528023598820059,
+              "pass@64": 0.9587020648967551,
+              "pass@128": 0.9646017699115044
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.996875,
+              "5": 0.9619047619047619,
+              "6": 0.9580838323353293,
+              "7": 0.8875739644970414,
+              "8": 0.8076923076923077,
+              "9": 0.7098591549295775,
+              "10": 0.6430678466076696
+            },
+            "pass@2": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9809523809523809,
+              "6": 0.9700598802395209,
+              "7": 0.9556213017751479,
+              "8": 0.8782051282051282,
+              "9": 0.7887323943661971,
+              "10": 0.696165191740413
+            },
+            "pass@4": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 0.9850299401197605,
+              "7": 0.9733727810650887,
+              "8": 0.9358974358974359,
+              "9": 0.8901408450704226,
+              "10": 0.7905604719764012
+            },
+            "pass@8": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 0.9910179640718563,
+              "7": 0.9970414201183432,
+              "8": 0.9775641025641025,
+              "9": 0.9323943661971831,
+              "10": 0.8790560471976401
+            },
+            "pass@16": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 0.9970059880239521,
+              "7": 1.0,
+              "8": 0.9871794871794872,
+              "9": 0.9774647887323944,
+              "10": 0.9351032448377581
+            },
+            "pass@32": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 0.9967948717948718,
+              "9": 0.9971830985915493,
+              "10": 0.9528023598820059
+            },
+            "pass@64": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 0.9971830985915493,
+              "10": 0.9587020648967551
+            },
+            "pass@128": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9968253968253968,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 0.9971830985915493,
+              "10": 0.9646017699115044
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.8821344140493077,
+            "pass@2": 0.9165822357311719,
+            "pass@4": 0.9510300574130361,
+            "pass@8": 0.9739952718676123,
+            "pass@16": 0.9878419452887538,
+            "pass@32": 0.9935832489023978,
+            "pass@64": 0.994596420128335,
+            "pass@128": 0.9952718676122931
+          }
+        },
+        "movie_festival_awards": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "3": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "4": {
+              "pass@1": 0.9912280701754386,
+              "pass@2": 0.9912280701754386,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "5": {
+              "pass@1": 0.9470588235294117,
+              "pass@2": 0.9823529411764705,
+              "pass@4": 0.9970588235294118,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "6": {
+              "pass@1": 0.9716088328075709,
+              "pass@2": 0.9779179810725552,
+              "pass@4": 0.9905362776025236,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "7": {
+              "pass@1": 0.879746835443038,
+              "pass@2": 0.930379746835443,
+              "pass@4": 0.9683544303797469,
+              "pass@8": 0.9841772151898734,
+              "pass@16": 0.9936708860759493,
+              "pass@32": 0.9968354430379747,
+              "pass@64": 0.9968354430379747,
+              "pass@128": 1.0
+            },
+            "8": {
+              "pass@1": 0.8018018018018018,
+              "pass@2": 0.8498498498498499,
+              "pass@4": 0.918918918918919,
+              "pass@8": 0.9669669669669669,
+              "pass@16": 0.990990990990991,
+              "pass@32": 0.996996996996997,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "9": {
+              "pass@1": 0.7620578778135049,
+              "pass@2": 0.8231511254019293,
+              "pass@4": 0.9067524115755627,
+              "pass@8": 0.9678456591639871,
+              "pass@16": 0.9807073954983923,
+              "pass@32": 0.9967845659163987,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "10": {
+              "pass@1": 0.6,
+              "pass@2": 0.6597014925373135,
+              "pass@4": 0.7492537313432835,
+              "pass@8": 0.8597014925373134,
+              "pass@16": 0.9194029850746268,
+              "pass@32": 0.9432835820895522,
+              "pass@64": 0.9641791044776119,
+              "pass@128": 0.9731343283582089
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9912280701754386,
+              "5": 0.9470588235294117,
+              "6": 0.9716088328075709,
+              "7": 0.879746835443038,
+              "8": 0.8018018018018018,
+              "9": 0.7620578778135049,
+              "10": 0.6
+            },
+            "pass@2": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9912280701754386,
+              "5": 0.9823529411764705,
+              "6": 0.9779179810725552,
+              "7": 0.930379746835443,
+              "8": 0.8498498498498499,
+              "9": 0.8231511254019293,
+              "10": 0.6597014925373135
+            },
+            "pass@4": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9970588235294118,
+              "6": 0.9905362776025236,
+              "7": 0.9683544303797469,
+              "8": 0.918918918918919,
+              "9": 0.9067524115755627,
+              "10": 0.7492537313432835
+            },
+            "pass@8": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 0.9841772151898734,
+              "8": 0.9669669669669669,
+              "9": 0.9678456591639871,
+              "10": 0.8597014925373134
+            },
+            "pass@16": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 0.9936708860759493,
+              "8": 0.990990990990991,
+              "9": 0.9807073954983923,
+              "10": 0.9194029850746268
+            },
+            "pass@32": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 0.9968354430379747,
+              "8": 0.996996996996997,
+              "9": 0.9967845659163987,
+              "10": 0.9432835820895522
+            },
+            "pass@64": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 0.9968354430379747,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9641791044776119
+            },
+            "pass@128": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9731343283582089
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.8855038500167393,
+            "pass@2": 0.9139604954804151,
+            "pass@4": 0.9484432541011047,
+            "pass@8": 0.9755607633076665,
+            "pass@16": 0.9872782055574155,
+            "pass@32": 0.9926347505858721,
+            "pass@64": 0.995647807164379,
+            "pass@128": 0.9969869434214932
+          }
+        },
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "3": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "4": {
+              "pass@1": 0.9792899408284024,
+              "pass@2": 0.9881656804733728,
+              "pass@4": 0.9940828402366864,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "5": {
+              "pass@1": 0.9681159420289855,
+              "pass@2": 0.9884057971014493,
+              "pass@4": 0.991304347826087,
+              "pass@8": 0.9971014492753624,
+              "pass@16": 0.9971014492753624,
+              "pass@32": 0.9971014492753624,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "6": {
+              "pass@1": 0.9742120343839542,
+              "pass@2": 0.9799426934097422,
+              "pass@4": 0.9885386819484241,
+              "pass@8": 0.9914040114613181,
+              "pass@16": 0.997134670487106,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "7": {
+              "pass@1": 0.8583815028901735,
+              "pass@2": 0.9132947976878613,
+              "pass@4": 0.9508670520231214,
+              "pass@8": 0.9884393063583815,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "8": {
+              "pass@1": 0.7971830985915493,
+              "pass@2": 0.8732394366197183,
+              "pass@4": 0.9577464788732394,
+              "pass@8": 0.9887323943661972,
+              "pass@16": 0.9971830985915493,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "9": {
+              "pass@1": 0.7425149700598802,
+              "pass@2": 0.8053892215568862,
+              "pass@4": 0.8922155688622755,
+              "pass@8": 0.9730538922155688,
+              "pass@16": 0.9970059880239521,
+              "pass@32": 0.9970059880239521,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "10": {
+              "pass@1": 0.656441717791411,
+              "pass@2": 0.7208588957055214,
+              "pass@4": 0.8006134969325154,
+              "pass@8": 0.8650306748466258,
+              "pass@16": 0.9079754601226994,
+              "pass@32": 0.9447852760736196,
+              "pass@64": 0.9478527607361963,
+              "pass@128": 0.950920245398773
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9792899408284024,
+              "5": 0.9681159420289855,
+              "6": 0.9742120343839542,
+              "7": 0.8583815028901735,
+              "8": 0.7971830985915493,
+              "9": 0.7425149700598802,
+              "10": 0.656441717791411
+            },
+            "pass@2": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9881656804733728,
+              "5": 0.9884057971014493,
+              "6": 0.9799426934097422,
+              "7": 0.9132947976878613,
+              "8": 0.8732394366197183,
+              "9": 0.8053892215568862,
+              "10": 0.7208588957055214
+            },
+            "pass@4": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9940828402366864,
+              "5": 0.991304347826087,
+              "6": 0.9885386819484241,
+              "7": 0.9508670520231214,
+              "8": 0.9577464788732394,
+              "9": 0.8922155688622755,
+              "10": 0.8006134969325154
+            },
+            "pass@8": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9971014492753624,
+              "6": 0.9914040114613181,
+              "7": 0.9884393063583815,
+              "8": 0.9887323943661972,
+              "9": 0.9730538922155688,
+              "10": 0.8650306748466258
+            },
+            "pass@16": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9971014492753624,
+              "6": 0.997134670487106,
+              "7": 1.0,
+              "8": 0.9971830985915493,
+              "9": 0.9970059880239521,
+              "10": 0.9079754601226994
+            },
+            "pass@32": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9971014492753624,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 0.9970059880239521,
+              "10": 0.9447852760736196
+            },
+            "pass@64": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9478527607361963
+            },
+            "pass@128": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.950920245398773
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.8866317169069463,
+            "pass@2": 0.919397116644823,
+            "pass@4": 0.953473132372215,
+            "pass@8": 0.97870249017038,
+            "pass@16": 0.9888597640891219,
+            "pass@32": 0.9934469200524246,
+            "pass@64": 0.994429882044561,
+            "pass@128": 0.9947575360419397
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.989,
+            "pass@2": 0.993,
+            "pass@4": 0.998,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.959,
+            "pass@2": 0.984,
+            "pass@4": 0.995,
+            "pass@8": 0.998,
+            "pass@16": 0.998,
+            "pass@32": 0.998,
+            "pass@64": 0.999,
+            "pass@128": 0.999
+          },
+          "6": {
+            "pass@1": 0.968,
+            "pass@2": 0.976,
+            "pass@4": 0.988,
+            "pass@8": 0.994,
+            "pass@16": 0.998,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.875,
+            "pass@2": 0.933,
+            "pass@4": 0.964,
+            "pass@8": 0.99,
+            "pass@16": 0.998,
+            "pass@32": 0.999,
+            "pass@64": 0.999,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.802,
+            "pass@2": 0.867,
+            "pass@4": 0.938,
+            "pass@8": 0.978,
+            "pass@16": 0.992,
+            "pass@32": 0.998,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.737,
+            "pass@2": 0.805,
+            "pass@4": 0.896,
+            "pass@8": 0.957,
+            "pass@16": 0.985,
+            "pass@32": 0.997,
+            "pass@64": 0.999,
+            "pass@128": 0.999
+          },
+          "10": {
+            "pass@1": 0.633,
+            "pass@2": 0.692,
+            "pass@4": 0.78,
+            "pass@8": 0.868,
+            "pass@16": 0.921,
+            "pass@32": 0.947,
+            "pass@64": 0.957,
+            "pass@128": 0.963
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 0.989,
+            "5": 0.959,
+            "6": 0.968,
+            "7": 0.875,
+            "8": 0.802,
+            "9": 0.737,
+            "10": 0.633
+          },
+          "pass@2": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 0.993,
+            "5": 0.984,
+            "6": 0.976,
+            "7": 0.933,
+            "8": 0.867,
+            "9": 0.805,
+            "10": 0.692
+          },
+          "pass@4": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 0.998,
+            "5": 0.995,
+            "6": 0.988,
+            "7": 0.964,
+            "8": 0.938,
+            "9": 0.896,
+            "10": 0.78
+          },
+          "pass@8": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.998,
+            "6": 0.994,
+            "7": 0.99,
+            "8": 0.978,
+            "9": 0.957,
+            "10": 0.868
+          },
+          "pass@16": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.998,
+            "6": 0.998,
+            "7": 0.998,
+            "8": 0.992,
+            "9": 0.985,
+            "10": 0.921
+          },
+          "pass@32": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.998,
+            "6": 1.0,
+            "7": 0.999,
+            "8": 0.998,
+            "9": 0.997,
+            "10": 0.947
+          },
+          "pass@64": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.999,
+            "6": 1.0,
+            "7": 0.999,
+            "8": 1.0,
+            "9": 0.999,
+            "10": 0.957
+          },
+          "pass@128": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.999,
+            "6": 1.0,
+            "7": 1.0,
+            "8": 1.0,
+            "9": 0.999,
+            "10": 0.963
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.8847777777777778,
+          "pass@2": 0.9166666666666666,
+          "pass@4": 0.951,
+          "pass@8": 0.9761111111111112,
+          "pass@16": 0.988,
+          "pass@32": 0.9932222222222222,
+          "pass@64": 0.9948888888888889,
+          "pass@128": 0.9956666666666667
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "crazy_zootopia": {
+            "per_op": {
+              "2": 321,
+              "3": 327,
+              "4": 320,
+              "5": 315,
+              "6": 334,
+              "7": 338,
+              "8": 312,
+              "9": 355,
+              "10": 339
+            },
+            "total": 2961
+          },
+          "movie_festival_awards": {
+            "per_op": {
+              "2": 350,
+              "3": 343,
+              "4": 342,
+              "5": 340,
+              "6": 317,
+              "7": 316,
+              "8": 333,
+              "9": 311,
+              "10": 335
+            },
+            "total": 2987
+          },
+          "teachers_in_school": {
+            "per_op": {
+              "2": 329,
+              "3": 330,
+              "4": 338,
+              "5": 345,
+              "6": 349,
+              "7": 346,
+              "8": 355,
+              "9": 334,
+              "10": 326
+            },
+            "total": 3052
+          }
+        },
+        "total": {
+          "per_op": {
+            "2": 1000,
+            "3": 1000,
+            "4": 1000,
+            "5": 1000,
+            "6": 1000,
+            "7": 1000,
+            "8": 1000,
+            "9": 1000,
+            "10": 1000
+          },
+          "total": 9000
+        }
+      }
+    },
+    "checkpoint-38646_ood_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "crazy_zootopia": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.6005665722379604,
+              "pass@2": 0.6742209631728046,
+              "pass@4": 0.773371104815864,
+              "pass@8": 0.8640226628895185,
+              "pass@16": 0.9178470254957507,
+              "pass@32": 0.9518413597733711,
+              "pass@64": 0.9745042492917847,
+              "pass@128": 0.9830028328611898
+            },
+            "12": {
+              "pass@1": 0.6638418079096046,
+              "pass@2": 0.7231638418079096,
+              "pass@4": 0.8022598870056498,
+              "pass@8": 0.8813559322033898,
+              "pass@16": 0.9293785310734464,
+              "pass@32": 0.9491525423728814,
+              "pass@64": 0.963276836158192,
+              "pass@128": 0.9745762711864406
+            },
+            "13": {
+              "pass@1": 0.5893416927899686,
+              "pass@2": 0.6394984326018809,
+              "pass@4": 0.7272727272727273,
+              "pass@8": 0.8056426332288401,
+              "pass@16": 0.8714733542319749,
+              "pass@32": 0.9122257053291536,
+              "pass@64": 0.9467084639498433,
+              "pass@128": 0.9717868338557993
+            },
+            "14": {
+              "pass@1": 0.5607476635514018,
+              "pass@2": 0.5919003115264797,
+              "pass@4": 0.6417445482866043,
+              "pass@8": 0.7227414330218068,
+              "pass@16": 0.7757009345794392,
+              "pass@32": 0.8286604361370716,
+              "pass@64": 0.8847352024922118,
+              "pass@128": 0.9283489096573209
+            },
+            "15": {
+              "pass@1": 0.5057803468208093,
+              "pass@2": 0.5549132947976878,
+              "pass@4": 0.5953757225433526,
+              "pass@8": 0.6647398843930635,
+              "pass@16": 0.7456647398843931,
+              "pass@32": 0.8179190751445087,
+              "pass@64": 0.9104046242774566,
+              "pass@128": 0.9450867052023122
+            },
+            "16": {
+              "pass@1": 0.4935483870967742,
+              "pass@2": 0.5193548387096775,
+              "pass@4": 0.5548387096774193,
+              "pass@8": 0.5870967741935483,
+              "pass@16": 0.6548387096774193,
+              "pass@32": 0.7032258064516129,
+              "pass@64": 0.7354838709677419,
+              "pass@128": 0.7709677419354839
+            },
+            "17": {
+              "pass@1": 0.5289017341040463,
+              "pass@2": 0.546242774566474,
+              "pass@4": 0.5809248554913294,
+              "pass@8": 0.6184971098265896,
+              "pass@16": 0.6705202312138728,
+              "pass@32": 0.7167630057803468,
+              "pass@64": 0.7803468208092486,
+              "pass@128": 0.8294797687861272
+            },
+            "18": {
+              "pass@1": 0.4,
+              "pass@2": 0.4142857142857143,
+              "pass@4": 0.4514285714285714,
+              "pass@8": 0.49714285714285716,
+              "pass@16": 0.5457142857142857,
+              "pass@32": 0.6057142857142858,
+              "pass@64": 0.6714285714285714,
+              "pass@128": 0.7314285714285714
+            },
+            "19": {
+              "pass@1": 0.3063063063063063,
+              "pass@2": 0.33633633633633636,
+              "pass@4": 0.39039039039039036,
+              "pass@8": 0.4174174174174174,
+              "pass@16": 0.45645645645645644,
+              "pass@32": 0.5015015015015015,
+              "pass@64": 0.5555555555555556,
+              "pass@128": 0.5825825825825826
+            },
+            "20": {
+              "pass@1": 0.21621621621621623,
+              "pass@2": 0.24924924924924924,
+              "pass@4": 0.2822822822822823,
+              "pass@8": 0.3153153153153153,
+              "pass@16": 0.34234234234234234,
+              "pass@32": 0.39039039039039036,
+              "pass@64": 0.43843843843843844,
+              "pass@128": 0.4924924924924925
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.6005665722379604,
+              "12": 0.6638418079096046,
+              "13": 0.5893416927899686,
+              "14": 0.5607476635514018,
+              "15": 0.5057803468208093,
+              "16": 0.4935483870967742,
+              "17": 0.5289017341040463,
+              "18": 0.4,
+              "19": 0.3063063063063063,
+              "20": 0.21621621621621623
+            },
+            "pass@2": {
+              "11": 0.6742209631728046,
+              "12": 0.7231638418079096,
+              "13": 0.6394984326018809,
+              "14": 0.5919003115264797,
+              "15": 0.5549132947976878,
+              "16": 0.5193548387096775,
+              "17": 0.546242774566474,
+              "18": 0.4142857142857143,
+              "19": 0.33633633633633636,
+              "20": 0.24924924924924924
+            },
+            "pass@4": {
+              "11": 0.773371104815864,
+              "12": 0.8022598870056498,
+              "13": 0.7272727272727273,
+              "14": 0.6417445482866043,
+              "15": 0.5953757225433526,
+              "16": 0.5548387096774193,
+              "17": 0.5809248554913294,
+              "18": 0.4514285714285714,
+              "19": 0.39039039039039036,
+              "20": 0.2822822822822823
+            },
+            "pass@8": {
+              "11": 0.8640226628895185,
+              "12": 0.8813559322033898,
+              "13": 0.8056426332288401,
+              "14": 0.7227414330218068,
+              "15": 0.6647398843930635,
+              "16": 0.5870967741935483,
+              "17": 0.6184971098265896,
+              "18": 0.49714285714285716,
+              "19": 0.4174174174174174,
+              "20": 0.3153153153153153
+            },
+            "pass@16": {
+              "11": 0.9178470254957507,
+              "12": 0.9293785310734464,
+              "13": 0.8714733542319749,
+              "14": 0.7757009345794392,
+              "15": 0.7456647398843931,
+              "16": 0.6548387096774193,
+              "17": 0.6705202312138728,
+              "18": 0.5457142857142857,
+              "19": 0.45645645645645644,
+              "20": 0.34234234234234234
+            },
+            "pass@32": {
+              "11": 0.9518413597733711,
+              "12": 0.9491525423728814,
+              "13": 0.9122257053291536,
+              "14": 0.8286604361370716,
+              "15": 0.8179190751445087,
+              "16": 0.7032258064516129,
+              "17": 0.7167630057803468,
+              "18": 0.6057142857142858,
+              "19": 0.5015015015015015,
+              "20": 0.39039039039039036
+            },
+            "pass@64": {
+              "11": 0.9745042492917847,
+              "12": 0.963276836158192,
+              "13": 0.9467084639498433,
+              "14": 0.8847352024922118,
+              "15": 0.9104046242774566,
+              "16": 0.7354838709677419,
+              "17": 0.7803468208092486,
+              "18": 0.6714285714285714,
+              "19": 0.5555555555555556,
+              "20": 0.43843843843843844
+            },
+            "pass@128": {
+              "11": 0.9830028328611898,
+              "12": 0.9745762711864406,
+              "13": 0.9717868338557993,
+              "14": 0.9283489096573209,
+              "15": 0.9450867052023122,
+              "16": 0.7709677419354839,
+              "17": 0.8294797687861272,
+              "18": 0.7314285714285714,
+              "19": 0.5825825825825826,
+              "20": 0.4924924924924925
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.487369985141159,
+            "pass@2": 0.5260029717682021,
+            "pass@4": 0.5812778603268945,
+            "pass@8": 0.6389301634472511,
+            "pass@16": 0.6924219910846954,
+            "pass@32": 0.7390787518573552,
+            "pass@64": 0.787518573551263,
+            "pass@128": 0.8222882615156017
+          }
+        },
+        "movie_festival_awards": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.6095238095238096,
+              "pass@2": 0.6952380952380952,
+              "pass@4": 0.8095238095238095,
+              "pass@8": 0.873015873015873,
+              "pass@16": 0.9333333333333333,
+              "pass@32": 0.9619047619047619,
+              "pass@64": 0.9841269841269841,
+              "pass@128": 0.9904761904761905
+            },
+            "12": {
+              "pass@1": 0.617737003058104,
+              "pass@2": 0.6788990825688074,
+              "pass@4": 0.7675840978593272,
+              "pass@8": 0.8837920489296636,
+              "pass@16": 0.9327217125382263,
+              "pass@32": 0.9724770642201835,
+              "pass@64": 0.981651376146789,
+              "pass@128": 0.9938837920489296
+            },
+            "13": {
+              "pass@1": 0.5864197530864198,
+              "pass@2": 0.6851851851851852,
+              "pass@4": 0.7623456790123457,
+              "pass@8": 0.8302469135802469,
+              "pass@16": 0.8796296296296297,
+              "pass@32": 0.9197530864197531,
+              "pass@64": 0.9506172839506173,
+              "pass@128": 0.9629629629629629
+            },
+            "14": {
+              "pass@1": 0.56973293768546,
+              "pass@2": 0.599406528189911,
+              "pass@4": 0.6528189910979229,
+              "pass@8": 0.7151335311572701,
+              "pass@16": 0.7833827893175074,
+              "pass@32": 0.8486646884272997,
+              "pass@64": 0.9287833827893175,
+              "pass@128": 0.9584569732937686
+            },
+            "15": {
+              "pass@1": 0.5,
+              "pass@2": 0.5308641975308642,
+              "pass@4": 0.5802469135802469,
+              "pass@8": 0.6481481481481481,
+              "pass@16": 0.7314814814814815,
+              "pass@32": 0.7962962962962963,
+              "pass@64": 0.8765432098765432,
+              "pass@128": 0.9320987654320988
+            },
+            "16": {
+              "pass@1": 0.5398230088495575,
+              "pass@2": 0.5781710914454278,
+              "pass@4": 0.6106194690265486,
+              "pass@8": 0.6460176991150443,
+              "pass@16": 0.7050147492625368,
+              "pass@32": 0.7463126843657817,
+              "pass@64": 0.7935103244837758,
+              "pass@128": 0.8230088495575221
+            },
+            "17": {
+              "pass@1": 0.4759036144578313,
+              "pass@2": 0.5090361445783133,
+              "pass@4": 0.536144578313253,
+              "pass@8": 0.5662650602409639,
+              "pass@16": 0.6234939759036144,
+              "pass@32": 0.7018072289156626,
+              "pass@64": 0.7680722891566265,
+              "pass@128": 0.7891566265060241
+            },
+            "18": {
+              "pass@1": 0.4290322580645161,
+              "pass@2": 0.45483870967741935,
+              "pass@4": 0.4870967741935484,
+              "pass@8": 0.535483870967742,
+              "pass@16": 0.5709677419354838,
+              "pass@32": 0.6387096774193548,
+              "pass@64": 0.7258064516129032,
+              "pass@128": 0.7741935483870968
+            },
+            "19": {
+              "pass@1": 0.2990936555891239,
+              "pass@2": 0.3564954682779456,
+              "pass@4": 0.3867069486404834,
+              "pass@8": 0.4199395770392749,
+              "pass@16": 0.459214501510574,
+              "pass@32": 0.4743202416918429,
+              "pass@64": 0.525679758308157,
+              "pass@128": 0.5770392749244713
+            },
+            "20": {
+              "pass@1": 0.1921921921921922,
+              "pass@2": 0.24324324324324326,
+              "pass@4": 0.2822822822822823,
+              "pass@8": 0.30930930930930933,
+              "pass@16": 0.3333333333333333,
+              "pass@32": 0.36036036036036034,
+              "pass@64": 0.4144144144144144,
+              "pass@128": 0.46846846846846846
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.6095238095238096,
+              "12": 0.617737003058104,
+              "13": 0.5864197530864198,
+              "14": 0.56973293768546,
+              "15": 0.5,
+              "16": 0.5398230088495575,
+              "17": 0.4759036144578313,
+              "18": 0.4290322580645161,
+              "19": 0.2990936555891239,
+              "20": 0.1921921921921922
+            },
+            "pass@2": {
+              "11": 0.6952380952380952,
+              "12": 0.6788990825688074,
+              "13": 0.6851851851851852,
+              "14": 0.599406528189911,
+              "15": 0.5308641975308642,
+              "16": 0.5781710914454278,
+              "17": 0.5090361445783133,
+              "18": 0.45483870967741935,
+              "19": 0.3564954682779456,
+              "20": 0.24324324324324326
+            },
+            "pass@4": {
+              "11": 0.8095238095238095,
+              "12": 0.7675840978593272,
+              "13": 0.7623456790123457,
+              "14": 0.6528189910979229,
+              "15": 0.5802469135802469,
+              "16": 0.6106194690265486,
+              "17": 0.536144578313253,
+              "18": 0.4870967741935484,
+              "19": 0.3867069486404834,
+              "20": 0.2822822822822823
+            },
+            "pass@8": {
+              "11": 0.873015873015873,
+              "12": 0.8837920489296636,
+              "13": 0.8302469135802469,
+              "14": 0.7151335311572701,
+              "15": 0.6481481481481481,
+              "16": 0.6460176991150443,
+              "17": 0.5662650602409639,
+              "18": 0.535483870967742,
+              "19": 0.4199395770392749,
+              "20": 0.30930930930930933
+            },
+            "pass@16": {
+              "11": 0.9333333333333333,
+              "12": 0.9327217125382263,
+              "13": 0.8796296296296297,
+              "14": 0.7833827893175074,
+              "15": 0.7314814814814815,
+              "16": 0.7050147492625368,
+              "17": 0.6234939759036144,
+              "18": 0.5709677419354838,
+              "19": 0.459214501510574,
+              "20": 0.3333333333333333
+            },
+            "pass@32": {
+              "11": 0.9619047619047619,
+              "12": 0.9724770642201835,
+              "13": 0.9197530864197531,
+              "14": 0.8486646884272997,
+              "15": 0.7962962962962963,
+              "16": 0.7463126843657817,
+              "17": 0.7018072289156626,
+              "18": 0.6387096774193548,
+              "19": 0.4743202416918429,
+              "20": 0.36036036036036034
+            },
+            "pass@64": {
+              "11": 0.9841269841269841,
+              "12": 0.981651376146789,
+              "13": 0.9506172839506173,
+              "14": 0.9287833827893175,
+              "15": 0.8765432098765432,
+              "16": 0.7935103244837758,
+              "17": 0.7680722891566265,
+              "18": 0.7258064516129032,
+              "19": 0.525679758308157,
+              "20": 0.4144144144144144
+            },
+            "pass@128": {
+              "11": 0.9904761904761905,
+              "12": 0.9938837920489296,
+              "13": 0.9629629629629629,
+              "14": 0.9584569732937686,
+              "15": 0.9320987654320988,
+              "16": 0.8230088495575221,
+              "17": 0.7891566265060241,
+              "18": 0.7741935483870968,
+              "19": 0.5770392749244713,
+              "20": 0.46846846846846846
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.48135696821515894,
+            "pass@2": 0.53239608801956,
+            "pass@4": 0.5864914425427873,
+            "pass@8": 0.6415036674816625,
+            "pass@16": 0.6940709046454768,
+            "pass@32": 0.7408312958435208,
+            "pass@64": 0.7937041564792175,
+            "pass@128": 0.8257946210268948
+          }
+        },
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.6686746987951807,
+              "pass@2": 0.7228915662650602,
+              "pass@4": 0.8042168674698795,
+              "pass@8": 0.8825301204819277,
+              "pass@16": 0.9216867469879518,
+              "pass@32": 0.9578313253012049,
+              "pass@64": 0.9819277108433735,
+              "pass@128": 0.9879518072289156
+            },
+            "12": {
+              "pass@1": 0.5987460815047022,
+              "pass@2": 0.658307210031348,
+              "pass@4": 0.7711598746081505,
+              "pass@8": 0.8369905956112853,
+              "pass@16": 0.9122257053291536,
+              "pass@32": 0.9467084639498433,
+              "pass@64": 0.9717868338557993,
+              "pass@128": 0.9811912225705329
+            },
+            "13": {
+              "pass@1": 0.6162464985994398,
+              "pass@2": 0.6750700280112045,
+              "pass@4": 0.7675070028011205,
+              "pass@8": 0.8487394957983193,
+              "pass@16": 0.9131652661064426,
+              "pass@32": 0.938375350140056,
+              "pass@64": 0.9635854341736695,
+              "pass@128": 0.9831932773109243
+            },
+            "14": {
+              "pass@1": 0.5672514619883041,
+              "pass@2": 0.6023391812865497,
+              "pass@4": 0.6374269005847953,
+              "pass@8": 0.716374269005848,
+              "pass@16": 0.7923976608187134,
+              "pass@32": 0.8713450292397661,
+              "pass@64": 0.9385964912280702,
+              "pass@128": 0.9590643274853801
+            },
+            "15": {
+              "pass@1": 0.5545454545454546,
+              "pass@2": 0.5848484848484848,
+              "pass@4": 0.6242424242424243,
+              "pass@8": 0.6818181818181818,
+              "pass@16": 0.7545454545454545,
+              "pass@32": 0.8121212121212121,
+              "pass@64": 0.8727272727272727,
+              "pass@128": 0.9272727272727272
+            },
+            "16": {
+              "pass@1": 0.5156695156695157,
+              "pass@2": 0.5384615384615384,
+              "pass@4": 0.5726495726495726,
+              "pass@8": 0.6296296296296297,
+              "pass@16": 0.6837606837606838,
+              "pass@32": 0.7492877492877493,
+              "pass@64": 0.7977207977207977,
+              "pass@128": 0.8319088319088319
+            },
+            "17": {
+              "pass@1": 0.43478260869565216,
+              "pass@2": 0.45652173913043476,
+              "pass@4": 0.5186335403726708,
+              "pass@8": 0.5403726708074534,
+              "pass@16": 0.6180124223602484,
+              "pass@32": 0.6770186335403726,
+              "pass@64": 0.7298136645962733,
+              "pass@128": 0.7888198757763976
+            },
+            "18": {
+              "pass@1": 0.39705882352941174,
+              "pass@2": 0.4323529411764706,
+              "pass@4": 0.45294117647058824,
+              "pass@8": 0.5,
+              "pass@16": 0.5323529411764706,
+              "pass@32": 0.5823529411764706,
+              "pass@64": 0.6529411764705882,
+              "pass@128": 0.7205882352941176
+            },
+            "19": {
+              "pass@1": 0.2976190476190476,
+              "pass@2": 0.3601190476190476,
+              "pass@4": 0.39880952380952384,
+              "pass@8": 0.4375,
+              "pass@16": 0.4732142857142857,
+              "pass@32": 0.5148809523809523,
+              "pass@64": 0.5625,
+              "pass@128": 0.6101190476190477
+            },
+            "20": {
+              "pass@1": 0.19760479041916168,
+              "pass@2": 0.24251497005988024,
+              "pass@4": 0.2934131736526946,
+              "pass@8": 0.32934131736526945,
+              "pass@16": 0.3532934131736527,
+              "pass@32": 0.3772455089820359,
+              "pass@64": 0.4251497005988024,
+              "pass@128": 0.46407185628742514
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.6686746987951807,
+              "12": 0.5987460815047022,
+              "13": 0.6162464985994398,
+              "14": 0.5672514619883041,
+              "15": 0.5545454545454546,
+              "16": 0.5156695156695157,
+              "17": 0.43478260869565216,
+              "18": 0.39705882352941174,
+              "19": 0.2976190476190476,
+              "20": 0.19760479041916168
+            },
+            "pass@2": {
+              "11": 0.7228915662650602,
+              "12": 0.658307210031348,
+              "13": 0.6750700280112045,
+              "14": 0.6023391812865497,
+              "15": 0.5848484848484848,
+              "16": 0.5384615384615384,
+              "17": 0.45652173913043476,
+              "18": 0.4323529411764706,
+              "19": 0.3601190476190476,
+              "20": 0.24251497005988024
+            },
+            "pass@4": {
+              "11": 0.8042168674698795,
+              "12": 0.7711598746081505,
+              "13": 0.7675070028011205,
+              "14": 0.6374269005847953,
+              "15": 0.6242424242424243,
+              "16": 0.5726495726495726,
+              "17": 0.5186335403726708,
+              "18": 0.45294117647058824,
+              "19": 0.39880952380952384,
+              "20": 0.2934131736526946
+            },
+            "pass@8": {
+              "11": 0.8825301204819277,
+              "12": 0.8369905956112853,
+              "13": 0.8487394957983193,
+              "14": 0.716374269005848,
+              "15": 0.6818181818181818,
+              "16": 0.6296296296296297,
+              "17": 0.5403726708074534,
+              "18": 0.5,
+              "19": 0.4375,
+              "20": 0.32934131736526945
+            },
+            "pass@16": {
+              "11": 0.9216867469879518,
+              "12": 0.9122257053291536,
+              "13": 0.9131652661064426,
+              "14": 0.7923976608187134,
+              "15": 0.7545454545454545,
+              "16": 0.6837606837606838,
+              "17": 0.6180124223602484,
+              "18": 0.5323529411764706,
+              "19": 0.4732142857142857,
+              "20": 0.3532934131736527
+            },
+            "pass@32": {
+              "11": 0.9578313253012049,
+              "12": 0.9467084639498433,
+              "13": 0.938375350140056,
+              "14": 0.8713450292397661,
+              "15": 0.8121212121212121,
+              "16": 0.7492877492877493,
+              "17": 0.6770186335403726,
+              "18": 0.5823529411764706,
+              "19": 0.5148809523809523,
+              "20": 0.3772455089820359
+            },
+            "pass@64": {
+              "11": 0.9819277108433735,
+              "12": 0.9717868338557993,
+              "13": 0.9635854341736695,
+              "14": 0.9385964912280702,
+              "15": 0.8727272727272727,
+              "16": 0.7977207977207977,
+              "17": 0.7298136645962733,
+              "18": 0.6529411764705882,
+              "19": 0.5625,
+              "20": 0.4251497005988024
+            },
+            "pass@128": {
+              "11": 0.9879518072289156,
+              "12": 0.9811912225705329,
+              "13": 0.9831932773109243,
+              "14": 0.9590643274853801,
+              "15": 0.9272727272727272,
+              "16": 0.8319088319088319,
+              "17": 0.7888198757763976,
+              "18": 0.7205882352941176,
+              "19": 0.6101190476190477,
+              "20": 0.46407185628742514
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.4852809991079393,
+            "pass@2": 0.527802557240559,
+            "pass@4": 0.584299732381802,
+            "pass@8": 0.6407969075230449,
+            "pass@16": 0.695807314897413,
+            "pass@32": 0.7430865298840321,
+            "pass@64": 0.7900683913172762,
+            "pass@128": 0.8257508177222718
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.626,
+            "pass@2": 0.697,
+            "pass@4": 0.795,
+            "pass@8": 0.873,
+            "pass@16": 0.924,
+            "pass@32": 0.957,
+            "pass@64": 0.98,
+            "pass@128": 0.987
+          },
+          "12": {
+            "pass@1": 0.628,
+            "pass@2": 0.688,
+            "pass@4": 0.781,
+            "pass@8": 0.868,
+            "pass@16": 0.925,
+            "pass@32": 0.956,
+            "pass@64": 0.972,
+            "pass@128": 0.983
+          },
+          "13": {
+            "pass@1": 0.598,
+            "pass@2": 0.667,
+            "pass@4": 0.753,
+            "pass@8": 0.829,
+            "pass@16": 0.889,
+            "pass@32": 0.924,
+            "pass@64": 0.954,
+            "pass@128": 0.973
+          },
+          "14": {
+            "pass@1": 0.566,
+            "pass@2": 0.598,
+            "pass@4": 0.644,
+            "pass@8": 0.718,
+            "pass@16": 0.784,
+            "pass@32": 0.85,
+            "pass@64": 0.918,
+            "pass@128": 0.949
+          },
+          "15": {
+            "pass@1": 0.52,
+            "pass@2": 0.557,
+            "pass@4": 0.6,
+            "pass@8": 0.665,
+            "pass@16": 0.744,
+            "pass@32": 0.809,
+            "pass@64": 0.887,
+            "pass@128": 0.935
+          },
+          "16": {
+            "pass@1": 0.517,
+            "pass@2": 0.546,
+            "pass@4": 0.58,
+            "pass@8": 0.622,
+            "pass@16": 0.682,
+            "pass@32": 0.734,
+            "pass@64": 0.777,
+            "pass@128": 0.81
+          },
+          "17": {
+            "pass@1": 0.481,
+            "pass@2": 0.505,
+            "pass@4": 0.546,
+            "pass@8": 0.576,
+            "pass@16": 0.638,
+            "pass@32": 0.699,
+            "pass@64": 0.76,
+            "pass@128": 0.803
+          },
+          "18": {
+            "pass@1": 0.408,
+            "pass@2": 0.433,
+            "pass@4": 0.463,
+            "pass@8": 0.51,
+            "pass@16": 0.549,
+            "pass@32": 0.608,
+            "pass@64": 0.682,
+            "pass@128": 0.741
+          },
+          "19": {
+            "pass@1": 0.301,
+            "pass@2": 0.351,
+            "pass@4": 0.392,
+            "pass@8": 0.425,
+            "pass@16": 0.463,
+            "pass@32": 0.497,
+            "pass@64": 0.548,
+            "pass@128": 0.59
+          },
+          "20": {
+            "pass@1": 0.202,
+            "pass@2": 0.245,
+            "pass@4": 0.286,
+            "pass@8": 0.318,
+            "pass@16": 0.343,
+            "pass@32": 0.376,
+            "pass@64": 0.426,
+            "pass@128": 0.475
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "11": 0.626,
+            "12": 0.628,
+            "13": 0.598,
+            "14": 0.566,
+            "15": 0.52,
+            "16": 0.517,
+            "17": 0.481,
+            "18": 0.408,
+            "19": 0.301,
+            "20": 0.202
+          },
+          "pass@2": {
+            "11": 0.697,
+            "12": 0.688,
+            "13": 0.667,
+            "14": 0.598,
+            "15": 0.557,
+            "16": 0.546,
+            "17": 0.505,
+            "18": 0.433,
+            "19": 0.351,
+            "20": 0.245
+          },
+          "pass@4": {
+            "11": 0.795,
+            "12": 0.781,
+            "13": 0.753,
+            "14": 0.644,
+            "15": 0.6,
+            "16": 0.58,
+            "17": 0.546,
+            "18": 0.463,
+            "19": 0.392,
+            "20": 0.286
+          },
+          "pass@8": {
+            "11": 0.873,
+            "12": 0.868,
+            "13": 0.829,
+            "14": 0.718,
+            "15": 0.665,
+            "16": 0.622,
+            "17": 0.576,
+            "18": 0.51,
+            "19": 0.425,
+            "20": 0.318
+          },
+          "pass@16": {
+            "11": 0.924,
+            "12": 0.925,
+            "13": 0.889,
+            "14": 0.784,
+            "15": 0.744,
+            "16": 0.682,
+            "17": 0.638,
+            "18": 0.549,
+            "19": 0.463,
+            "20": 0.343
+          },
+          "pass@32": {
+            "11": 0.957,
+            "12": 0.956,
+            "13": 0.924,
+            "14": 0.85,
+            "15": 0.809,
+            "16": 0.734,
+            "17": 0.699,
+            "18": 0.608,
+            "19": 0.497,
+            "20": 0.376
+          },
+          "pass@64": {
+            "11": 0.98,
+            "12": 0.972,
+            "13": 0.954,
+            "14": 0.918,
+            "15": 0.887,
+            "16": 0.777,
+            "17": 0.76,
+            "18": 0.682,
+            "19": 0.548,
+            "20": 0.426
+          },
+          "pass@128": {
+            "11": 0.987,
+            "12": 0.983,
+            "13": 0.973,
+            "14": 0.949,
+            "15": 0.935,
+            "16": 0.81,
+            "17": 0.803,
+            "18": 0.741,
+            "19": 0.59,
+            "20": 0.475
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.4847,
+          "pass@2": 0.5287,
+          "pass@4": 0.584,
+          "pass@8": 0.6404,
+          "pass@16": 0.6941,
+          "pass@32": 0.741,
+          "pass@64": 0.7904,
+          "pass@128": 0.8246
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "crazy_zootopia": {
+            "per_op": {
+              "11": 353,
+              "12": 354,
+              "13": 319,
+              "14": 321,
+              "15": 346,
+              "16": 310,
+              "17": 346,
+              "18": 350,
+              "19": 333,
+              "20": 333
+            },
+            "total": 3365
+          },
+          "movie_festival_awards": {
+            "per_op": {
+              "11": 315,
+              "12": 327,
+              "13": 324,
+              "14": 337,
+              "15": 324,
+              "16": 339,
+              "17": 332,
+              "18": 310,
+              "19": 331,
+              "20": 333
+            },
+            "total": 3272
+          },
+          "teachers_in_school": {
+            "per_op": {
+              "11": 332,
+              "12": 319,
+              "13": 357,
+              "14": 342,
+              "15": 330,
+              "16": 351,
+              "17": 322,
+              "18": 340,
+              "19": 336,
+              "20": 334
+            },
+            "total": 3363
+          }
+        },
+        "total": {
+          "per_op": {
+            "11": 1000,
+            "12": 1000,
+            "13": 1000,
+            "14": 1000,
+            "15": 1000,
+            "16": 1000,
+            "17": 1000,
+            "18": 1000,
+            "19": 1000,
+            "20": 1000
+          },
+          "total": 10000
+        }
+      }
+    }
+  }
+}

Qwen3-100M-midtrain-diff_ar-block16/composition/summary.csv

@@ -0,0 +1,2 @@
+checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
+checkpoint-38646,0.9932248263888889,0.83598125,0.9104650493421053,0.0,0.0,0.0,200.3281736111111,378.03591796875,1152000,1280000,2432000

Qwen3-100M-midtrain-diff_ar-block16/composition/summary.json

@@ -0,0 +1,645 @@
+[
+  {
+    "checkpoint": "checkpoint-38646",
+    "id_acc": 0.9932248263888889,
+    "ood_acc": 0.83598125,
+    "total_acc": 0.9104650493421053,
+    "id_avg_loss": 0.0,
+    "ood_avg_loss": 0.0,
+    "total_avg_loss": 0.0,
+    "id_avg_resp_len": 200.3281736111111,
+    "ood_avg_resp_len": 378.03591796875,
+    "count_id": 1152000,
+    "count_ood": 1280000,
+    "count_total": 2432000,
+    "template_metrics": {
+      "crazy_zootopia": {
+        "count": 809728,
+        "correct": 735827,
+        "answer_accuracy": 0.9087335500316156,
+        "avg_response_len": 280.81283097533986,
+        "resp_tokens_sum": 227382012,
+        "pass_at_k": {
+          "pass@1": 0.9087335500316156,
+          "pass@2": 0.9306044600025811,
+          "pass@4": 0.9464375050603238,
+          "pass@8": 0.9578451948086137,
+          "pass@16": 0.9661519899120048,
+          "pass@32": 0.9723665816849368,
+          "pass@64": 0.9774353954902267,
+          "pass@128": 0.9818210559595321
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9643482669616519,
+            "pass@2": 0.9724997386941677,
+            "pass@4": 0.9787036025917114,
+            "pass@8": 0.9837184467724707,
+            "pass@16": 0.9876176547055514,
+            "pass@32": 0.990048743242238,
+            "pass@64": 0.9918928791392225,
+            "pass@128": 0.9941002949852508
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9998805428134556,
+            "pass@2": 0.9999977425413568,
+            "pass@4": 0.9999999997133384,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9967757936507936,
+            "pass@2": 0.9988044463192103,
+            "pass@4": 0.9997930303354938,
+            "pass@8": 0.9999940728357846,
+            "pass@16": 0.999999996928095,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9981521332335329,
+            "pass@2": 0.999078737210618,
+            "pass@4": 0.9997273029868273,
+            "pass@8": 0.9999777200738017,
+            "pass@16": 0.9999998973084382,
+            "pass@32": 0.999999999999679,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.998959874260355,
+            "pass@2": 0.9998143607603783,
+            "pass@4": 0.9999912155026479,
+            "pass@8": 0.9999999836752375,
+            "pass@16": 0.9999999999999905,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9960687099358975,
+            "pass@2": 0.9985512252675145,
+            "pass@4": 0.9996649123787412,
+            "pass@8": 0.9999730867256917,
+            "pass@16": 0.9999998586577866,
+            "pass@32": 0.9999999999993815,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9940801056338028,
+            "pass@2": 0.9950447765332151,
+            "pass@4": 0.9956578631896364,
+            "pass@8": 0.996375867480201,
+            "pass@16": 0.9969724240620078,
+            "pass@32": 0.9971729386969103,
+            "pass@64": 0.9971830959162251,
+            "pass@128": 0.9971830985915493
+          },
+          "11": {
+            "pass@1": 0.9711623583569405,
+            "pass@2": 0.9805191998840088,
+            "pass@4": 0.986303336286222,
+            "pass@8": 0.989850641477996,
+            "pass@16": 0.99240868699634,
+            "pass@32": 0.994293234140082,
+            "pass@64": 0.9956686613210934,
+            "pass@128": 0.9971671388101983
+          },
+          "12": {
+            "pass@1": 0.9658368644067796,
+            "pass@2": 0.979869405889942,
+            "pass@4": 0.9899045508612269,
+            "pass@8": 0.9949535165900368,
+            "pass@16": 0.9969359713595375,
+            "pass@32": 0.9982218320963844,
+            "pass@64": 0.9992952648675736,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9498187695924765,
+            "pass@2": 0.9693292893639082,
+            "pass@4": 0.9815207164543299,
+            "pass@8": 0.9885514952344922,
+            "pass@16": 0.993030758542598,
+            "pass@32": 0.9957866897580077,
+            "pass@64": 0.9968002718717178,
+            "pass@128": 0.9968652037617555
+          },
+          "14": {
+            "pass@1": 0.9244548286604362,
+            "pass@2": 0.9464950971373904,
+            "pass@4": 0.962845547343965,
+            "pass@8": 0.9744775229835171,
+            "pass@16": 0.9831241464032126,
+            "pass@32": 0.9892894037314296,
+            "pass@64": 0.9925585252418944,
+            "pass@128": 0.9937694704049844
+          },
+          "15": {
+            "pass@1": 0.9763367052023122,
+            "pass@2": 0.9890843327568163,
+            "pass@4": 0.995195050131017,
+            "pass@8": 0.9987062171803248,
+            "pass@16": 0.9998947926322176,
+            "pass@32": 0.9999995163740522,
+            "pass@64": 0.9999999999997273,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.8670362903225807,
+            "pass@2": 0.9042973710947418,
+            "pass@4": 0.9324377792493678,
+            "pass@8": 0.9511663793058849,
+            "pass@16": 0.9632457765868611,
+            "pass@32": 0.972380137786845,
+            "pass@64": 0.9803184368887754,
+            "pass@128": 0.9870967741935484
+          },
+          "17": {
+            "pass@1": 0.8796739523121387,
+            "pass@2": 0.9130583354194169,
+            "pass@4": 0.9346565702950422,
+            "pass@8": 0.949022076996675,
+            "pass@16": 0.9577801252538675,
+            "pass@32": 0.9642535207391926,
+            "pass@64": 0.9711417638827387,
+            "pass@128": 0.976878612716763
+          },
+          "18": {
+            "pass@1": 0.7535491071428572,
+            "pass@2": 0.8142751687289084,
+            "pass@4": 0.854199288124698,
+            "pass@8": 0.8827058936508151,
+            "pass@16": 0.9034225204407647,
+            "pass@32": 0.9183155907983432,
+            "pass@64": 0.9304605804851537,
+            "pass@128": 0.9428571428571428
+          },
+          "19": {
+            "pass@1": 0.5902073948948949,
+            "pass@2": 0.6823130955758907,
+            "pass@4": 0.7493216109810598,
+            "pass@8": 0.7968110404034479,
+            "pass@16": 0.8325350334716398,
+            "pass@32": 0.859961150550141,
+            "pass@64": 0.8827473329655091,
+            "pass@128": 0.9009009009009009
+          },
+          "20": {
+            "pass@1": 0.4485735735735736,
+            "pass@2": 0.5452467723629141,
+            "pass@4": 0.6280490980294128,
+            "pass@8": 0.6976146762314849,
+            "pass@16": 0.7542370813489142,
+            "pass@32": 0.799241057750153,
+            "pass@64": 0.8368248092112288,
+            "pass@128": 0.8708708708708709
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 821120,
+        "correct": 751099,
+        "answer_accuracy": 0.9147250097427904,
+        "avg_response_len": 290.0505114964926,
+        "resp_tokens_sum": 238166276,
+        "pass_at_k": {
+          "pass@1": 0.9147250097427904,
+          "pass@2": 0.9368544058892373,
+          "pass@4": 0.9526741754464987,
+          "pass@8": 0.9638412119281327,
+          "pass@16": 0.9716894634319847,
+          "pass@32": 0.9774218920691676,
+          "pass@64": 0.9817877251624777,
+          "pass@128": 0.9851909586905689
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9476131134969326,
+            "pass@2": 0.9578519002705184,
+            "pass@4": 0.9660574474778076,
+            "pass@8": 0.9719991236070697,
+            "pass@16": 0.9763583249372905,
+            "pass@32": 0.980656765738378,
+            "pass@64": 0.9852951841811889,
+            "pass@128": 0.99079754601227
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9982717803030303,
+            "pass@2": 0.9992185635886424,
+            "pass@4": 0.9998076788696866,
+            "pass@8": 0.9999893019044129,
+            "pass@16": 0.9999999789617312,
+            "pass@32": 0.9999999999999926,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.9999768860946746,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9955842391304348,
+            "pass@2": 0.9970807657195024,
+            "pass@4": 0.9974464667460045,
+            "pass@8": 0.9977680565376104,
+            "pass@16": 0.9983124881128989,
+            "pass@32": 0.999097406302473,
+            "pass@64": 0.9998273650576287,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9994627507163324,
+            "pass@2": 0.9999027028856349,
+            "pass@4": 0.9999971459513118,
+            "pass@8": 0.9999999985260104,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9984420158959537,
+            "pass@2": 0.9991658037868099,
+            "pass@4": 0.9997657894822396,
+            "pass@8": 0.999983097148883,
+            "pass@16": 0.9999999407613452,
+            "pass@32": 0.9999999999999076,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9972711267605634,
+            "pass@2": 0.9988802401020296,
+            "pass@4": 0.9997421626874106,
+            "pass@8": 0.9999832931204404,
+            "pass@16": 0.9999999422581453,
+            "pass@32": 0.99999999999991,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9965147829341318,
+            "pass@2": 0.9973931320430004,
+            "pass@4": 0.9978456220292824,
+            "pass@8": 0.998468082786387,
+            "pass@16": 0.9992530482905866,
+            "pass@32": 0.999851122735485,
+            "pass@64": 0.9999980007921191,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9768213478915663,
+            "pass@2": 0.982426507209942,
+            "pass@4": 0.9875867705919291,
+            "pass@8": 0.9916699109297928,
+            "pass@16": 0.9941198985224184,
+            "pass@32": 0.9952745122110707,
+            "pass@64": 0.9962408632343975,
+            "pass@128": 0.9969879518072289
+          },
+          "12": {
+            "pass@1": 0.965492750783699,
+            "pass@2": 0.9797927517093281,
+            "pass@4": 0.9877445484870819,
+            "pass@8": 0.9923678770900078,
+            "pass@16": 0.9955227193860775,
+            "pass@32": 0.9973880317416322,
+            "pass@64": 0.9984300583339356,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9623599439775911,
+            "pass@2": 0.9779697804318578,
+            "pass@4": 0.9872410336838148,
+            "pass@8": 0.9920194344744313,
+            "pass@16": 0.9942491977157041,
+            "pass@32": 0.9950794358980206,
+            "pass@64": 0.9957983155432268,
+            "pass@128": 0.9971988795518207
+          },
+          "14": {
+            "pass@1": 0.9557291666666666,
+            "pass@2": 0.9763930618870013,
+            "pass@4": 0.9898799393496869,
+            "pass@8": 0.9971921128374414,
+            "pass@16": 0.9996866455629769,
+            "pass@32": 0.9999957420781709,
+            "pass@64": 0.9999999999150341,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.9709043560606061,
+            "pass@2": 0.9843362264376043,
+            "pass@4": 0.9913319070911589,
+            "pass@8": 0.9957115636967746,
+            "pass@16": 0.9984277498659098,
+            "pass@32": 0.9997408383719552,
+            "pass@64": 0.9999955357422065,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.9107015669515669,
+            "pass@2": 0.9377124133521775,
+            "pass@4": 0.9564160948898481,
+            "pass@8": 0.9692630161138946,
+            "pass@16": 0.9776624723599159,
+            "pass@32": 0.9832541934776856,
+            "pass@64": 0.9874615955124632,
+            "pass@128": 0.9914529914529915
+          },
+          "17": {
+            "pass@1": 0.8672360248447205,
+            "pass@2": 0.9112839750330124,
+            "pass@4": 0.9387636921937549,
+            "pass@8": 0.9558194693071742,
+            "pass@16": 0.9657225480978907,
+            "pass@32": 0.9716417279156446,
+            "pass@64": 0.9759516660247486,
+            "pass@128": 0.9782608695652174
+          },
+          "18": {
+            "pass@1": 0.7839613970588235,
+            "pass@2": 0.8523347035664656,
+            "pass@4": 0.8947464600013231,
+            "pass@8": 0.9214377697528552,
+            "pass@16": 0.9388117402083257,
+            "pass@32": 0.951005420422579,
+            "pass@64": 0.9607119551856902,
+            "pass@128": 0.9705882352941176
+          },
+          "19": {
+            "pass@1": 0.5933547247023809,
+            "pass@2": 0.6915281830005624,
+            "pass@4": 0.766301561356169,
+            "pass@8": 0.8222817559953494,
+            "pass@16": 0.8649021620408612,
+            "pass@32": 0.8975421565193381,
+            "pass@64": 0.9221817078073432,
+            "pass@128": 0.9404761904761905
+          },
+          "20": {
+            "pass@1": 0.4492889221556886,
+            "pass@2": 0.5484648321467304,
+            "pass@4": 0.6333525578202419,
+            "pass@8": 0.7015890174857582,
+            "pass@16": 0.7546736568184237,
+            "pass@32": 0.796908044139665,
+            "pass@64": 0.829107778372377,
+            "pass@128": 0.8502994011976048
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 801152,
+        "correct": 727325,
+        "answer_accuracy": 0.9078489475155775,
+        "avg_response_len": 310.94691519212336,
+        "resp_tokens_sum": 249115743,
+        "pass_at_k": {
+          "pass@1": 0.9078489475155775,
+          "pass@2": 0.9314184621389209,
+          "pass@4": 0.9479047528888704,
+          "pass@8": 0.959567070014469,
+          "pass@16": 0.9676303174059113,
+          "pass@32": 0.9731903126693562,
+          "pass@64": 0.9775192818766577,
+          "pass@128": 0.9816264579006231
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9642024253731343,
+            "pass@2": 0.9741138059701494,
+            "pass@4": 0.9819364163434796,
+            "pass@8": 0.9881400872944226,
+            "pass@16": 0.9925379015562921,
+            "pass@32": 0.9955510391342886,
+            "pass@64": 0.997754923360079,
+            "pass@128": 1.0
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9912913602941177,
+            "pass@2": 0.9954405974988421,
+            "pass@4": 0.9979701380158362,
+            "pass@8": 0.9993477897991202,
+            "pass@16": 0.9999126470598457,
+            "pass@32": 0.999998726614258,
+            "pass@64": 0.999999999987784,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999507097791798,
+            "pass@2": 0.999999611888025,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9933742088607594,
+            "pass@2": 0.9947567558307585,
+            "pass@4": 0.9957734057965857,
+            "pass@8": 0.9969989697228501,
+            "pass@16": 0.9982483505948174,
+            "pass@32": 0.9992648536542246,
+            "pass@64": 0.9999088007454459,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9931493993993994,
+            "pass@2": 0.9975064877869996,
+            "pass@4": 0.999478633301468,
+            "pass@8": 0.9999675398986609,
+            "pass@16": 0.9999998926590724,
+            "pass@32": 0.9999999999998089,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9930667202572347,
+            "pass@2": 0.9967814011190724,
+            "pass@4": 0.9985162387257863,
+            "pass@8": 0.9994198002264989,
+            "pass@16": 0.9999085821695214,
+            "pass@32": 0.999998609246854,
+            "pass@64": 0.9999999999866449,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.963640873015873,
+            "pass@2": 0.9752730127484065,
+            "pass@4": 0.9833094119187484,
+            "pass@8": 0.9885337281484384,
+            "pass@16": 0.9929572101573484,
+            "pass@32": 0.9967672908887808,
+            "pass@64": 0.9991148007644227,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9675793195718655,
+            "pass@2": 0.9831443087962632,
+            "pass@4": 0.9927234121537561,
+            "pass@8": 0.9962327927192676,
+            "pass@16": 0.9969067136543522,
+            "pass@32": 0.9969418396245668,
+            "pass@64": 0.9969418960244639,
+            "pass@128": 0.9969418960244648
+          },
+          "13": {
+            "pass@1": 0.953125,
+            "pass@2": 0.9724694243948675,
+            "pass@4": 0.9839528441467964,
+            "pass@8": 0.9914051147315478,
+            "pass@16": 0.9962751879157801,
+            "pass@32": 0.9989514073218897,
+            "pass@64": 0.9999069142994649,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9672663204747775,
+            "pass@2": 0.9819023224841703,
+            "pass@4": 0.9891655304029129,
+            "pass@8": 0.9926525053330164,
+            "pass@16": 0.9941520139636356,
+            "pass@32": 0.9947959873034944,
+            "pass@64": 0.9955489586054439,
+            "pass@128": 0.9970326409495549
+          },
+          "15": {
+            "pass@1": 0.9480372299382716,
+            "pass@2": 0.9670028768105379,
+            "pass@4": 0.9780621108704005,
+            "pass@8": 0.9842947803211456,
+            "pass@16": 0.9879716058879758,
+            "pass@32": 0.9901270313479683,
+            "pass@64": 0.9919070327284573,
+            "pass@128": 0.9938271604938271
+          },
+          "16": {
+            "pass@1": 0.882973820058997,
+            "pass@2": 0.9149514842171274,
+            "pass@4": 0.9361787777080962,
+            "pass@8": 0.9505317245196182,
+            "pass@16": 0.95963740909398,
+            "pass@32": 0.9664879570651409,
+            "pass@64": 0.9730817511351202,
+            "pass@128": 0.9793510324483776
+          },
+          "17": {
+            "pass@1": 0.8233951430722891,
+            "pass@2": 0.8709084616023144,
+            "pass@4": 0.900955382553837,
+            "pass@8": 0.920503593025735,
+            "pass@16": 0.9343316545762004,
+            "pass@32": 0.9442573518899536,
+            "pass@64": 0.9519856298182777,
+            "pass@128": 0.9608433734939759
+          },
+          "18": {
+            "pass@1": 0.7824344758064516,
+            "pass@2": 0.8496388430276858,
+            "pass@4": 0.8933080521184847,
+            "pass@8": 0.9212389467048724,
+            "pass@16": 0.9380265394631339,
+            "pass@32": 0.9476456964449388,
+            "pass@64": 0.9534177140834851,
+            "pass@128": 0.9580645161290322
+          },
+          "19": {
+            "pass@1": 0.5686367069486404,
+            "pass@2": 0.6704094012417636,
+            "pass@4": 0.7465739156094909,
+            "pass@8": 0.8053303353516325,
+            "pass@16": 0.8499870368405819,
+            "pass@32": 0.8823623062494177,
+            "pass@64": 0.907953968913336,
+            "pass@128": 0.9305135951661632
+          },
+          "20": {
+            "pass@1": 0.4545561186186186,
+            "pass@2": 0.5521487284528623,
+            "pass@4": 0.6327929909662195,
+            "pass@8": 0.6981138753732286,
+            "pass@16": 0.7452411820752879,
+            "pass@32": 0.7785809400350618,
+            "pass@64": 0.8062433242479462,
+            "pass@128": 0.8348348348348348
+          }
+        }
+      }
+    }
+  }
+]

Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_id_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad599636b8c33c4e7935cbaeff8d20f8a960498bbfaef9961b706eaa1c1595f1
+size 1789731521

Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_metrics.json

@@ -0,0 +1,1897 @@
+{
+  "checkpoint": "checkpoint-38646",
+  "id": {
+    "answer_accuracy": 0.9930946180555555,
+    "avg_response_len": 200.28550694444445,
+    "avg_loss": 0.0,
+    "count": 1152000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.9930946180555555,
+      "pass@2": 0.9949989473972002,
+      "pass@4": 0.9963422400845728,
+      "pass@8": 0.9972624951869838,
+      "pass@16": 0.9979487555732529,
+      "pass@32": 0.9985491774462739,
+      "pass@64": 0.9991352162559932,
+      "pass@128": 0.9996666666666667
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.9540625,
+        "pass@2": 0.9639723179133853,
+        "pass@4": 0.9723782580614919,
+        "pass@8": 0.9787925265522301,
+        "pass@16": 0.9837687276335041,
+        "pass@32": 0.9881571311082412,
+        "pass@64": 0.9927600328966237,
+        "pass@128": 0.997
+      },
+      "2": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "3": {
+        "pass@1": 0.99946875,
+        "pass@2": 0.9998220964566928,
+        "pass@4": 0.9999744520997375,
+        "pass@8": 0.9999994736310946,
+        "pass@16": 0.9999999998889773,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "4": {
+        "pass@1": 0.999890625,
+        "pass@2": 0.9999948326771654,
+        "pass@4": 0.9999999880952382,
+        "pass@8": 0.9999999999999937,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "5": {
+        "pass@1": 0.9944765625,
+        "pass@2": 0.9970152559055121,
+        "pass@4": 0.9982060622422197,
+        "pass@8": 0.9988426718080923,
+        "pass@16": 0.9993716905884307,
+        "pass@32": 0.9998027430992852,
+        "pass@64": 0.9999860378787945,
+        "pass@128": 1.0
+      },
+      "6": {
+        "pass@1": 0.9999140625,
+        "pass@2": 0.9999985236220473,
+        "pass@4": 0.9999999999062616,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "7": {
+        "pass@1": 0.99834375,
+        "pass@2": 0.9991849163385828,
+        "pass@4": 0.9996791599175103,
+        "pass@8": 0.9999399336710925,
+        "pass@16": 0.9999976797361912,
+        "pass@32": 0.9999999983745138,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "8": {
+        "pass@1": 0.9961796875,
+        "pass@2": 0.9982639025590552,
+        "pass@4": 0.9995023362392201,
+        "pass@8": 0.999940916408823,
+        "pass@16": 0.9999986572834463,
+        "pass@32": 0.9999999995728239,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "9": {
+        "pass@1": 0.995515625,
+        "pass@2": 0.9967386811023622,
+        "pass@4": 0.9973399041994752,
+        "pass@8": 0.9978469346115266,
+        "pass@16": 0.9984020450287275,
+        "pass@32": 0.9989827248615993,
+        "pass@64": 0.9994708755285203,
+        "pass@128": 1.0
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.9540625,
+      "2": 1.0,
+      "3": 0.99946875,
+      "4": 0.999890625,
+      "5": 0.9944765625,
+      "6": 0.9999140625,
+      "7": 0.99834375,
+      "8": 0.9961796875,
+      "9": 0.995515625
+    },
+    "per_op_avg_response_len": {
+      "10": 281.6685703125,
+      "2": 115.6646328125,
+      "3": 144.9749609375,
+      "4": 159.97440625,
+      "5": 173.51565625,
+      "6": 198.98446875,
+      "7": 224.252796875,
+      "8": 243.7776015625,
+      "9": 259.75646875
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 281.6685703125,
+      "2": 115.6646328125,
+      "3": 144.9749609375,
+      "4": 159.97440625,
+      "5": 173.51565625,
+      "6": 198.98446875,
+      "7": 224.252796875,
+      "8": 243.7776015625,
+      "9": 259.75646875
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 379008,
+        "correct": 376657,
+        "answer_accuracy": 0.9937969647078689,
+        "avg_response_len": 190.8121912993921,
+        "resp_tokens_sum": 72319347,
+        "pass_at_k": {
+          "pass@1": 0.9937969647078689,
+          "pass@2": 0.9952953437336292,
+          "pass@4": 0.9963559719042212,
+          "pass@8": 0.9972608428511113,
+          "pass@16": 0.9980648749134898,
+          "pass@32": 0.9986967786412437,
+          "pass@64": 0.9991892593315244,
+          "pass@128": 0.999662276258021
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.954484697640118,
+            "pass@2": 0.9629707860079438,
+            "pass@4": 0.9709464638269774,
+            "pass@8": 0.9782191819839355,
+            "pass@16": 0.984593484638989,
+            "pass@32": 0.9892985145351701,
+            "pass@64": 0.9930035863666121,
+            "pass@128": 0.9970501474926253
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9980902777777778,
+            "pass@2": 0.9997808867641542,
+            "pass@4": 0.9999950205629058,
+            "pass@8": 0.9999999974305525,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9998830464071856,
+            "pass@2": 0.9999977898533642,
+            "pass@4": 0.9999999997193464,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9991910133136095,
+            "pass@2": 0.9999253803056423,
+            "pass@4": 0.9999992966855474,
+            "pass@8": 0.9999999999707041,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9977964743589743,
+            "pass@2": 0.9990386192711489,
+            "pass@4": 0.9997245203123646,
+            "pass@8": 0.9999788364839716,
+            "pass@16": 0.9999999148409177,
+            "pass@32": 0.9999999999998113,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9962367957746479,
+            "pass@2": 0.9972319646223802,
+            "pass@4": 0.9975970072402921,
+            "pass@8": 0.9979708642982112,
+            "pass@16": 0.9985716335094632,
+            "pass@32": 0.9993491975474896,
+            "pass@64": 0.9999188200066541,
+            "pass@128": 1.0
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 390656,
+        "correct": 387649,
+        "answer_accuracy": 0.9923026908584535,
+        "avg_response_len": 198.26654652686761,
+        "resp_tokens_sum": 77454016,
+        "pass_at_k": {
+          "pass@1": 0.9923026908584535,
+          "pass@2": 0.9942880249558829,
+          "pass@4": 0.9956853823809375,
+          "pass@8": 0.9965860027278677,
+          "pass@16": 0.997238113916091,
+          "pass@32": 0.9979263130447362,
+          "pass@64": 0.9987312934569264,
+          "pass@128": 0.9993446920052425
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9433713573619632,
+            "pass@2": 0.9535782540698516,
+            "pass@4": 0.9618546829460732,
+            "pass@8": 0.968359742151891,
+            "pass@16": 0.9741511694787096,
+            "pass@32": 0.9805862212217205,
+            "pass@64": 0.9881224160445984,
+            "pass@128": 0.9938650306748467
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9983901515151515,
+            "pass@2": 0.999460898353615,
+            "pass@4": 0.9999225821204167,
+            "pass@8": 0.9999984049427112,
+            "pass@16": 0.9999999996635676,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.999791974852071,
+            "pass@2": 0.9999868960536739,
+            "pass@4": 0.9999999650561431,
+            "pass@8": 0.9999999999999813,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9935914855072464,
+            "pass@2": 0.9972134257674314,
+            "pass@4": 0.9991986126734157,
+            "pass@8": 0.9998860811105759,
+            "pass@16": 0.9999969943734291,
+            "pass@32": 0.9999999992176973,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999328438395415,
+            "pass@2": 0.99999894242267,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9981484826589595,
+            "pass@2": 0.9991423353511447,
+            "pass@4": 0.9997656553768929,
+            "pass@8": 0.9999830971478822,
+            "pass@16": 0.9999999407613452,
+            "pass@32": 0.9999999999999076,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9978213028169014,
+            "pass@2": 0.998696212709327,
+            "pass@4": 0.9994122448602376,
+            "pass@8": 0.9998848700630322,
+            "pass@16": 0.9999964254002791,
+            "pass@32": 0.9999999987973975,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9976609281437125,
+            "pass@2": 0.9988149930454053,
+            "pass@4": 0.9995776498184733,
+            "pass@8": 0.9999453661202221,
+            "pass@16": 0.9999993082440582,
+            "pass@32": 0.9999999999733661,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 382336,
+        "correct": 379739,
+        "answer_accuracy": 0.9932075451958486,
+        "avg_response_len": 211.73925813943757,
+        "resp_tokens_sum": 80955541,
+        "pass_at_k": {
+          "pass@1": 0.9932075451958486,
+          "pass@2": 0.9954315238078923,
+          "pass@4": 0.9969997793525728,
+          "pass@8": 0.9979553467275731,
+          "pass@16": 0.9985597528853449,
+          "pass@32": 0.9990392795604974,
+          "pass@64": 0.9994943561408621,
+          "pass@128": 1.0
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9640391791044776,
+            "pass@2": 0.9751006287460335,
+            "pass@4": 0.9840680005484389,
+            "pass@8": 0.989525217666148,
+            "pass@16": 0.9922932987785924,
+            "pass@32": 0.9943696315299635,
+            "pass@64": 0.9970265954859788,
+            "pass@128": 1.0
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.9998857821637427,
+            "pass@2": 0.9999978415527007,
+            "pass@4": 0.9999999997259115,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9920266544117647,
+            "pass@2": 0.9942518961324689,
+            "pass@4": 0.9955414982134585,
+            "pass@8": 0.9967116901009395,
+            "pass@16": 0.998155080969405,
+            "pass@32": 0.9994198334387632,
+            "pass@64": 0.9999589349376307,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999260646687698,
+            "pass@2": 0.9999988356640751,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9976513053797469,
+            "pass@2": 0.9984395245689226,
+            "pass@4": 0.9992420280930073,
+            "pass@8": 0.9998284242652762,
+            "pass@16": 0.9999927222555876,
+            "pass@32": 0.9999999948561573,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9929147897897898,
+            "pass@2": 0.9970771706746117,
+            "pass@4": 0.9993902071362702,
+            "pass@8": 0.9999651367971397,
+            "pass@16": 0.9999998583663093,
+            "pass@32": 0.9999999999994196,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9923884646302251,
+            "pass@2": 0.9939457427146365,
+            "pass@4": 0.994643188391644,
+            "pass@8": 0.995451850487162,
+            "pass@16": 0.9964930745316808,
+            "pass@32": 0.9974718962737498,
+            "pass@64": 0.9983913004056533,
+            "pass@128": 1.0
+          }
+        }
+      }
+    }
+  },
+  "ood": {
+    "answer_accuracy": 0.870721875,
+    "avg_response_len": 379.828709375,
+    "avg_loss": 0.0,
+    "count": 1280000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.870721875,
+      "pass@2": 0.9070165846456469,
+      "pass@4": 0.9310222992875821,
+      "pass@8": 0.9473439413551321,
+      "pass@16": 0.9587791727410775,
+      "pass@32": 0.9673418489786182,
+      "pass@64": 0.9738290079881573,
+      "pass@128": 0.9788
+    },
+    "per_op_pass_at_k": {
+      "11": {
+        "pass@1": 0.9725234375,
+        "pass@2": 0.9814163385826772,
+        "pass@4": 0.9876329908136482,
+        "pass@8": 0.9918884097903063,
+        "pass@16": 0.9949111139255706,
+        "pass@32": 0.9969289474459831,
+        "pass@64": 0.9981950900579236,
+        "pass@128": 0.999
+      },
+      "12": {
+        "pass@1": 0.9633828125,
+        "pass@2": 0.9799354084645666,
+        "pass@4": 0.9903667950881146,
+        "pass@8": 0.9952444450104463,
+        "pass@16": 0.9974553279156712,
+        "pass@32": 0.9988006564676271,
+        "pass@64": 0.9996719396956972,
+        "pass@128": 1.0
+      },
+      "13": {
+        "pass@1": 0.9512578125,
+        "pass@2": 0.9713690944881891,
+        "pass@4": 0.9840455555868022,
+        "pass@8": 0.9908545724750295,
+        "pass@16": 0.9943492242524582,
+        "pass@32": 0.9960756104075091,
+        "pass@64": 0.9969796143219964,
+        "pass@128": 0.998
+      },
+      "14": {
+        "pass@1": 0.9520390625,
+        "pass@2": 0.9693324311023619,
+        "pass@4": 0.9814703537682793,
+        "pass@8": 0.9892758228409231,
+        "pass@16": 0.9936319146442153,
+        "pass@32": 0.9960982517605781,
+        "pass@64": 0.9977339665394206,
+        "pass@128": 0.999
+      },
+      "15": {
+        "pass@1": 0.959546875,
+        "pass@2": 0.9762183809055123,
+        "pass@4": 0.9848403532995874,
+        "pass@8": 0.9897748141534157,
+        "pass@16": 0.9926387666188844,
+        "pass@32": 0.9944456451139438,
+        "pass@64": 0.9959166661010433,
+        "pass@128": 0.997
+      },
+      "16": {
+        "pass@1": 0.896828125,
+        "pass@2": 0.9233396899606302,
+        "pass@4": 0.9414247777465318,
+        "pass@8": 0.953458790870688,
+        "pass@16": 0.9612533257789929,
+        "pass@32": 0.9673482732504611,
+        "pass@64": 0.9725713413565941,
+        "pass@128": 0.977
+      },
+      "17": {
+        "pass@1": 0.8799609375,
+        "pass@2": 0.9119715797244108,
+        "pass@4": 0.9323001194225726,
+        "pass@8": 0.9465483790840621,
+        "pass@16": 0.9567859939940381,
+        "pass@32": 0.9648555078508166,
+        "pass@64": 0.9711329473830459,
+        "pass@128": 0.976
+      },
+      "18": {
+        "pass@1": 0.83540625,
+        "pass@2": 0.8872344980314973,
+        "pass@4": 0.9175925875515554,
+        "pass@8": 0.9373478636623114,
+        "pass@16": 0.9512725593857325,
+        "pass@32": 0.9618185543165673,
+        "pass@64": 0.9696181077106221,
+        "pass@128": 0.975
+      },
+      "19": {
+        "pass@1": 0.6875390625,
+        "pass@2": 0.7691173720472443,
+        "pass@4": 0.8240086892575941,
+        "pass@8": 0.8627536267659003,
+        "pass@16": 0.890500855679011,
+        "pass@32": 0.9109176601230866,
+        "pass@64": 0.9261058523936584,
+        "pass@128": 0.938
+      },
+      "20": {
+        "pass@1": 0.608734375,
+        "pass@2": 0.7002310531496069,
+        "pass@4": 0.7665407703412074,
+        "pass@8": 0.8162926888982398,
+        "pass@16": 0.854992645216184,
+        "pass@32": 0.88612938304957,
+        "pass@64": 0.9103645543215643,
+        "pass@128": 0.929
+      }
+    },
+    "per_op_accuracy": {
+      "11": 0.9725234375,
+      "12": 0.9633828125,
+      "13": 0.9512578125,
+      "14": 0.9520390625,
+      "15": 0.959546875,
+      "16": 0.896828125,
+      "17": 0.8799609375,
+      "18": 0.83540625,
+      "19": 0.6875390625,
+      "20": 0.608734375
+    },
+    "per_op_avg_response_len": {
+      "11": 301.0858671875,
+      "12": 313.5365234375,
+      "13": 334.6242578125,
+      "14": 361.643125,
+      "15": 381.6772421875,
+      "16": 390.87915625,
+      "17": 412.801421875,
+      "18": 429.3104140625,
+      "19": 428.1585078125,
+      "20": 444.570578125
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "11": 301.0858671875,
+      "12": 313.5365234375,
+      "13": 334.6242578125,
+      "14": 361.643125,
+      "15": 381.6772421875,
+      "16": 390.87915625,
+      "17": 412.801421875,
+      "18": 429.3104140625,
+      "19": 428.1585078125,
+      "20": 444.570578125
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 430720,
+        "correct": 371057,
+        "answer_accuracy": 0.8614807763744428,
+        "avg_response_len": 361.376033153789,
+        "resp_tokens_sum": 155651885,
+        "pass_at_k": {
+          "pass@1": 0.8614807763744428,
+          "pass@2": 0.8980265601783103,
+          "pass@4": 0.9228763205676597,
+          "pass@8": 0.9399765617471983,
+          "pass@16": 0.9521830805726366,
+          "pass@32": 0.9618381392489881,
+          "pass@64": 0.9695388585332727,
+          "pass@128": 0.975334323922734
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9682852337110481,
+            "pass@2": 0.977452822823493,
+            "pass@4": 0.9842284054082472,
+            "pass@8": 0.9891870968251745,
+            "pass@16": 0.9931120473781875,
+            "pass@32": 0.9965309204049758,
+            "pass@64": 0.9991380576749798,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9647775423728814,
+            "pass@2": 0.9793001245607014,
+            "pass@4": 0.9885038096403199,
+            "pass@8": 0.9930489687895803,
+            "pass@16": 0.9957842683111874,
+            "pass@32": 0.9978504659297166,
+            "pass@64": 0.9992600866784821,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9408797021943573,
+            "pass@2": 0.9649356379433759,
+            "pass@4": 0.9808571087704949,
+            "pass@8": 0.9881229406714634,
+            "pass@16": 0.99055864455524,
+            "pass@32": 0.9913680560096522,
+            "pass@64": 0.9921630087220765,
+            "pass@128": 0.9937304075235109
+          },
+          "14": {
+            "pass@1": 0.9360640576323987,
+            "pass@2": 0.9562122856722352,
+            "pass@4": 0.9707606814919159,
+            "pass@8": 0.9804801905602375,
+            "pass@16": 0.9866715910082771,
+            "pass@32": 0.9909012287981317,
+            "pass@64": 0.9945214531172304,
+            "pass@128": 0.9968847352024922
+          },
+          "15": {
+            "pass@1": 0.9660404624277457,
+            "pass@2": 0.9819844198033775,
+            "pass@4": 0.989337811097312,
+            "pass@8": 0.9927511350152156,
+            "pass@16": 0.9945051007958996,
+            "pass@32": 0.9954700452049894,
+            "pass@64": 0.996392965878996,
+            "pass@128": 0.9971098265895953
+          },
+          "16": {
+            "pass@1": 0.8782762096774194,
+            "pass@2": 0.9086745142240289,
+            "pass@4": 0.9307146733674418,
+            "pass@8": 0.9456327971297461,
+            "pass@16": 0.9552193362938507,
+            "pass@32": 0.9633343766463572,
+            "pass@64": 0.9715414297733572,
+            "pass@128": 0.9806451612903225
+          },
+          "17": {
+            "pass@1": 0.8934248554913294,
+            "pass@2": 0.9237297175822676,
+            "pass@4": 0.9422482222236664,
+            "pass@8": 0.9540871391786083,
+            "pass@16": 0.9620315216397886,
+            "pass@32": 0.9683743703109818,
+            "pass@64": 0.9725996832538545,
+            "pass@128": 0.9739884393063584
+          },
+          "18": {
+            "pass@1": 0.8005357142857142,
+            "pass@2": 0.8531492547806522,
+            "pass@4": 0.8860468795864797,
+            "pass@8": 0.9091291155842222,
+            "pass@16": 0.9275554499538204,
+            "pass@32": 0.9442031158786668,
+            "pass@64": 0.9575396478067062,
+            "pass@128": 0.9657142857142857
+          },
+          "19": {
+            "pass@1": 0.6647428678678678,
+            "pass@2": 0.7474041462722564,
+            "pass@4": 0.8052284390689899,
+            "pass@8": 0.8453281707689558,
+            "pass@16": 0.8732022792173314,
+            "pass@32": 0.8947298964394246,
+            "pass@64": 0.9117678285429259,
+            "pass@128": 0.924924924924925
+          },
+          "20": {
+            "pass@1": 0.5938203828828829,
+            "pass@2": 0.6811821664183864,
+            "pass@4": 0.7462476041846112,
+            "pass@8": 0.7985336624704658,
+            "pass@16": 0.8402692553045812,
+            "pass@32": 0.8729885399962004,
+            "pass@64": 0.8982468595231101,
+            "pass@128": 0.918918918918919
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 430464,
+        "correct": 378048,
+        "answer_accuracy": 0.8782337198929527,
+        "avg_response_len": 375.42510175066906,
+        "resp_tokens_sum": 161606991,
+        "pass_at_k": {
+          "pass@1": 0.8782337198929527,
+          "pass@2": 0.9133105372031476,
+          "pass@4": 0.936427513771977,
+          "pass@8": 0.9522564458294169,
+          "pass@16": 0.9631596491448439,
+          "pass@32": 0.9708932145154232,
+          "pass@64": 0.9767694340645282,
+          "pass@128": 0.9815640796907523
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9791274472891566,
+            "pass@2": 0.9844943257281095,
+            "pass@4": 0.9885222114554959,
+            "pass@8": 0.9915842526776513,
+            "pass@16": 0.9936573596958077,
+            "pass@32": 0.9946784629745986,
+            "pass@64": 0.9954818436151764,
+            "pass@128": 0.9969879518072289
+          },
+          "12": {
+            "pass@1": 0.9556720219435737,
+            "pass@2": 0.9739207939426853,
+            "pass@4": 0.9861632180381843,
+            "pass@8": 0.9932283805113136,
+            "pass@16": 0.9967175800031935,
+            "pass@32": 0.9986257066354457,
+            "pass@64": 0.9997926928260641,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9692752100840336,
+            "pass@2": 0.9822620977524867,
+            "pass@4": 0.9899195350056031,
+            "pass@8": 0.9950258375460176,
+            "pass@16": 0.9984359616586164,
+            "pass@32": 0.9998247453476256,
+            "pass@64": 0.9999989636921891,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9544956140350878,
+            "pass@2": 0.9714041706957683,
+            "pass@4": 0.9850686652107085,
+            "pass@8": 0.9941410138686707,
+            "pass@16": 0.9982549357238066,
+            "pass@32": 0.9997482243275714,
+            "pass@64": 0.9999976075223646,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.962405303030303,
+            "pass@2": 0.9801527827487472,
+            "pass@4": 0.9897256777561895,
+            "pass@8": 0.9949554819709671,
+            "pass@16": 0.9975806792718727,
+            "pass@32": 0.9989957428151091,
+            "pass@64": 0.999819329898887,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.9157540954415955,
+            "pass@2": 0.9383107460125177,
+            "pass@4": 0.9528681665860143,
+            "pass@8": 0.9618255529076115,
+            "pass@16": 0.9666002487818389,
+            "pass@32": 0.9691287175972142,
+            "pass@64": 0.9713401996282022,
+            "pass@128": 0.9743589743589743
+          },
+          "17": {
+            "pass@1": 0.8866459627329193,
+            "pass@2": 0.9140307869125057,
+            "pass@4": 0.9320510835796145,
+            "pass@8": 0.9470251241563546,
+            "pass@16": 0.958828933724438,
+            "pass@32": 0.9669829599693496,
+            "pass@64": 0.9726542282720683,
+            "pass@128": 0.9782608695652174
+          },
+          "18": {
+            "pass@1": 0.8592830882352941,
+            "pass@2": 0.9078276980083376,
+            "pass@4": 0.9323671411477975,
+            "pass@8": 0.9477004466579106,
+            "pass@16": 0.9584144161429146,
+            "pass@32": 0.9654229587125491,
+            "pass@64": 0.9701825020374101,
+            "pass@128": 0.9735294117647059
+          },
+          "19": {
+            "pass@1": 0.6903366815476191,
+            "pass@2": 0.7712544232752157,
+            "pass@4": 0.8257569664841,
+            "pass@8": 0.8643543757323078,
+            "pass@16": 0.8929841094671906,
+            "pass@32": 0.914312749232494,
+            "pass@64": 0.9313348502509535,
+            "pass@128": 0.9464285714285714
+          },
+          "20": {
+            "pass@1": 0.6061938622754491,
+            "pass@2": 0.7069592360554483,
+            "pass@4": 0.779909319718268,
+            "pass@8": 0.8312956919046384,
+            "pass@16": 0.8691291374054729,
+            "pass@32": 0.9006392983887458,
+            "pass@64": 0.926877074969234,
+            "pass@128": 0.9461077844311377
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 418816,
+        "correct": 365419,
+        "answer_accuracy": 0.8725048708740831,
+        "avg_response_len": 403.3319452933985,
+        "resp_tokens_sum": 168921872,
+        "pass_at_k": {
+          "pass@1": 0.8725048708740831,
+          "pass@2": 0.9097931341205792,
+          "pass@4": 0.9338442681389322,
+          "pass@8": 0.9498715941160362,
+          "pass@16": 0.9610604404675287,
+          "pass@32": 0.9693518553783383,
+          "pass@64": 0.9752188918576026,
+          "pass@128": 0.9795232273838631
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9703125,
+            "pass@2": 0.9826138920134985,
+            "pass@4": 0.9905110777819439,
+            "pass@8": 0.9952361673715535,
+            "pass@16": 0.9982486278795625,
+            "pass@32": 0.9997469296363813,
+            "pass@64": 0.9999980432330696,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9693950688073395,
+            "pass@2": 0.9864906149919334,
+            "pass@4": 0.9964843422668495,
+            "pass@8": 0.999587940904666,
+            "pass@16": 0.9999840639525752,
+            "pass@32": 0.9999999728189605,
+            "pass@64": 0.9999999999999997,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9416232638888888,
+            "pass@2": 0.9657007934772044,
+            "pass@4": 0.9807125428765847,
+            "pass@8": 0.9889479333237797,
+            "pass@16": 0.9935783281704044,
+            "pass@32": 0.9965795260843454,
+            "pass@64": 0.9983950138936485,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9647626112759644,
+            "pass@2": 0.9797271840463564,
+            "pass@4": 0.988019856104782,
+            "pass@8": 0.992716483465879,
+            "pass@16": 0.9955701659139958,
+            "pass@32": 0.9973444053304108,
+            "pass@64": 0.9984967309084293,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.9497010030864198,
+            "pass@2": 0.9660535597112858,
+            "pass@4": 0.9750617191369597,
+            "pass@8": 0.9813197913201593,
+            "pass@16": 0.9856122764931645,
+            "pass@32": 0.9887173282223192,
+            "pass@64": 0.991433089630488,
+            "pass@128": 0.9938271604938271
+          },
+          "16": {
+            "pass@1": 0.8941970870206489,
+            "pass@2": 0.9212493177014375,
+            "pass@4": 0.9393702139555561,
+            "pass@8": 0.9519523737165061,
+            "pass@16": 0.9612349386592156,
+            "pass@32": 0.9691753292432682,
+            "pass@64": 0.9747878703756769,
+            "pass@128": 0.976401179941003
+          },
+          "17": {
+            "pass@1": 0.8594455948795181,
+            "pass@2": 0.897720433900958,
+            "pass@4": 0.922174053073938,
+            "pass@8": 0.9382293341864976,
+            "pass@16": 0.9493378639982005,
+            "pass@32": 0.959124887388815,
+            "pass@64": 0.9681289020289342,
+            "pass@128": 0.9759036144578314
+          },
+          "18": {
+            "pass@1": 0.8485887096774194,
+            "pass@2": 0.903131746888494,
+            "pass@4": 0.9370043603420539,
+            "pass@8": 0.9578532946585311,
+            "pass@16": 0.9702169368171149,
+            "pass@32": 0.9777537348282815,
+            "pass@64": 0.9826360654372754,
+            "pass@128": 0.9870967741935484
+          },
+          "19": {
+            "pass@1": 0.7076331193353474,
+            "pass@2": 0.7887924625924787,
+            "pass@4": 0.8411277290300948,
+            "pass@8": 0.878659443062788,
+            "pass@16": 0.9053831900865986,
+            "pass@32": 0.9237568606242907,
+            "pass@64": 0.9352225250892545,
+            "pass@128": 0.9425981873111783
+          },
+          "20": {
+            "pass@1": 0.626196509009009,
+            "pass@2": 0.7125315522215124,
+            "pass@4": 0.7734252413268156,
+            "pass@8": 0.8190036582565382,
+            "pass@16": 0.8555370910100033,
+            "pass@32": 0.8847167374444287,
+            "pass@64": 0.9059201413232556,
+            "pass@128": 0.9219219219219219
+          }
+        }
+      }
+    }
+  },
+  "total": {
+    "answer_accuracy": 0.9286879111842106,
+    "avg_loss": 0.0,
+    "count": 2432000,
+    "validation_metrics": {
+      "id": {},
+      "ood": {}
+    },
+    "pass_at_k": {
+      "pass@1": 0.9286879111842106,
+      "pass@2": 0.9486924406858649,
+      "pass@4": 0.9619633238756299,
+      "pass@8": 0.970989572117595,
+      "pass@16": 0.9773331856615709,
+      "pass@32": 0.9821242677264577,
+      "pass@64": 0.985816159272922,
+      "pass@128": 0.9886842105263158
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.9540625,
+        "pass@2": 0.9639723179133853,
+        "pass@4": 0.9723782580614919,
+        "pass@8": 0.9787925265522301,
+        "pass@16": 0.9837687276335041,
+        "pass@32": 0.9881571311082412,
+        "pass@64": 0.9927600328966237,
+        "pass@128": 0.997
+      },
+      "2": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "3": {
+        "pass@1": 0.99946875,
+        "pass@2": 0.9998220964566928,
+        "pass@4": 0.9999744520997375,
+        "pass@8": 0.9999994736310946,
+        "pass@16": 0.9999999998889773,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "4": {
+        "pass@1": 0.999890625,
+        "pass@2": 0.9999948326771654,
+        "pass@4": 0.9999999880952382,
+        "pass@8": 0.9999999999999937,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "5": {
+        "pass@1": 0.9944765625,
+        "pass@2": 0.9970152559055121,
+        "pass@4": 0.9982060622422197,
+        "pass@8": 0.9988426718080923,
+        "pass@16": 0.9993716905884307,
+        "pass@32": 0.9998027430992852,
+        "pass@64": 0.9999860378787945,
+        "pass@128": 1.0
+      },
+      "6": {
+        "pass@1": 0.9999140625,
+        "pass@2": 0.9999985236220473,
+        "pass@4": 0.9999999999062616,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "7": {
+        "pass@1": 0.99834375,
+        "pass@2": 0.9991849163385828,
+        "pass@4": 0.9996791599175103,
+        "pass@8": 0.9999399336710925,
+        "pass@16": 0.9999976797361912,
+        "pass@32": 0.9999999983745138,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "8": {
+        "pass@1": 0.9961796875,
+        "pass@2": 0.9982639025590552,
+        "pass@4": 0.9995023362392201,
+        "pass@8": 0.999940916408823,
+        "pass@16": 0.9999986572834463,
+        "pass@32": 0.9999999995728239,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "9": {
+        "pass@1": 0.995515625,
+        "pass@2": 0.9967386811023622,
+        "pass@4": 0.9973399041994752,
+        "pass@8": 0.9978469346115266,
+        "pass@16": 0.9984020450287275,
+        "pass@32": 0.9989827248615993,
+        "pass@64": 0.9994708755285203,
+        "pass@128": 1.0
+      },
+      "11": {
+        "pass@1": 0.9725234375,
+        "pass@2": 0.9814163385826772,
+        "pass@4": 0.9876329908136482,
+        "pass@8": 0.9918884097903063,
+        "pass@16": 0.9949111139255706,
+        "pass@32": 0.9969289474459831,
+        "pass@64": 0.9981950900579236,
+        "pass@128": 0.999
+      },
+      "12": {
+        "pass@1": 0.9633828125,
+        "pass@2": 0.9799354084645666,
+        "pass@4": 0.9903667950881146,
+        "pass@8": 0.9952444450104463,
+        "pass@16": 0.9974553279156712,
+        "pass@32": 0.9988006564676271,
+        "pass@64": 0.9996719396956972,
+        "pass@128": 1.0
+      },
+      "13": {
+        "pass@1": 0.9512578125,
+        "pass@2": 0.9713690944881891,
+        "pass@4": 0.9840455555868022,
+        "pass@8": 0.9908545724750295,
+        "pass@16": 0.9943492242524582,
+        "pass@32": 0.9960756104075091,
+        "pass@64": 0.9969796143219964,
+        "pass@128": 0.998
+      },
+      "14": {
+        "pass@1": 0.9520390625,
+        "pass@2": 0.9693324311023619,
+        "pass@4": 0.9814703537682793,
+        "pass@8": 0.9892758228409231,
+        "pass@16": 0.9936319146442153,
+        "pass@32": 0.9960982517605781,
+        "pass@64": 0.9977339665394206,
+        "pass@128": 0.999
+      },
+      "15": {
+        "pass@1": 0.959546875,
+        "pass@2": 0.9762183809055123,
+        "pass@4": 0.9848403532995874,
+        "pass@8": 0.9897748141534157,
+        "pass@16": 0.9926387666188844,
+        "pass@32": 0.9944456451139438,
+        "pass@64": 0.9959166661010433,
+        "pass@128": 0.997
+      },
+      "16": {
+        "pass@1": 0.896828125,
+        "pass@2": 0.9233396899606302,
+        "pass@4": 0.9414247777465318,
+        "pass@8": 0.953458790870688,
+        "pass@16": 0.9612533257789929,
+        "pass@32": 0.9673482732504611,
+        "pass@64": 0.9725713413565941,
+        "pass@128": 0.977
+      },
+      "17": {
+        "pass@1": 0.8799609375,
+        "pass@2": 0.9119715797244108,
+        "pass@4": 0.9323001194225726,
+        "pass@8": 0.9465483790840621,
+        "pass@16": 0.9567859939940381,
+        "pass@32": 0.9648555078508166,
+        "pass@64": 0.9711329473830459,
+        "pass@128": 0.976
+      },
+      "18": {
+        "pass@1": 0.83540625,
+        "pass@2": 0.8872344980314973,
+        "pass@4": 0.9175925875515554,
+        "pass@8": 0.9373478636623114,
+        "pass@16": 0.9512725593857325,
+        "pass@32": 0.9618185543165673,
+        "pass@64": 0.9696181077106221,
+        "pass@128": 0.975
+      },
+      "19": {
+        "pass@1": 0.6875390625,
+        "pass@2": 0.7691173720472443,
+        "pass@4": 0.8240086892575941,
+        "pass@8": 0.8627536267659003,
+        "pass@16": 0.890500855679011,
+        "pass@32": 0.9109176601230866,
+        "pass@64": 0.9261058523936584,
+        "pass@128": 0.938
+      },
+      "20": {
+        "pass@1": 0.608734375,
+        "pass@2": 0.7002310531496069,
+        "pass@4": 0.7665407703412074,
+        "pass@8": 0.8162926888982398,
+        "pass@16": 0.854992645216184,
+        "pass@32": 0.88612938304957,
+        "pass@64": 0.9103645543215643,
+        "pass@128": 0.929
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.9540625,
+      "2": 1.0,
+      "3": 0.99946875,
+      "4": 0.999890625,
+      "5": 0.9944765625,
+      "6": 0.9999140625,
+      "7": 0.99834375,
+      "8": 0.9961796875,
+      "9": 0.995515625,
+      "11": 0.9725234375,
+      "12": 0.9633828125,
+      "13": 0.9512578125,
+      "14": 0.9520390625,
+      "15": 0.959546875,
+      "16": 0.896828125,
+      "17": 0.8799609375,
+      "18": 0.83540625,
+      "19": 0.6875390625,
+      "20": 0.608734375
+    },
+    "per_op_avg_response_len": {
+      "10": 281.6685703125,
+      "2": 115.6646328125,
+      "3": 144.9749609375,
+      "4": 159.97440625,
+      "5": 173.51565625,
+      "6": 198.98446875,
+      "7": 224.252796875,
+      "8": 243.7776015625,
+      "9": 259.75646875,
+      "11": 301.0858671875,
+      "12": 313.5365234375,
+      "13": 334.6242578125,
+      "14": 361.643125,
+      "15": 381.6772421875,
+      "16": 390.87915625,
+      "17": 412.801421875,
+      "18": 429.3104140625,
+      "19": 428.1585078125,
+      "20": 444.570578125
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 281.6685703125,
+      "2": 115.6646328125,
+      "3": 144.9749609375,
+      "4": 159.97440625,
+      "5": 173.51565625,
+      "6": 198.98446875,
+      "7": 224.252796875,
+      "8": 243.7776015625,
+      "9": 259.75646875,
+      "11": 301.0858671875,
+      "12": 313.5365234375,
+      "13": 334.6242578125,
+      "14": 361.643125,
+      "15": 381.6772421875,
+      "16": 390.87915625,
+      "17": 412.801421875,
+      "18": 429.3104140625,
+      "19": 428.1585078125,
+      "20": 444.570578125
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 809728,
+        "correct": 747714,
+        "answer_accuracy": 0.9234137883338602,
+        "avg_response_len": 281.5405074296554,
+        "resp_tokens_sum": 227971232,
+        "pass_at_k": {
+          "pass@1": 0.9234137883338602,
+          "pass@2": 0.9435549933283615,
+          "pass@4": 0.9572698152890534,
+          "pass@8": 0.9667895172243828,
+          "pass@16": 0.9736588935734707,
+          "pass@32": 0.9790904995462497,
+          "pass@64": 0.9834172709208217,
+          "pass@128": 0.9867214669617452
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.954484697640118,
+            "pass@2": 0.9629707860079438,
+            "pass@4": 0.9709464638269774,
+            "pass@8": 0.9782191819839355,
+            "pass@16": 0.984593484638989,
+            "pass@32": 0.9892985145351701,
+            "pass@64": 0.9930035863666121,
+            "pass@128": 0.9970501474926253
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9980902777777778,
+            "pass@2": 0.9997808867641542,
+            "pass@4": 0.9999950205629058,
+            "pass@8": 0.9999999974305525,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9998830464071856,
+            "pass@2": 0.9999977898533642,
+            "pass@4": 0.9999999997193464,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9991910133136095,
+            "pass@2": 0.9999253803056423,
+            "pass@4": 0.9999992966855474,
+            "pass@8": 0.9999999999707041,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9977964743589743,
+            "pass@2": 0.9990386192711489,
+            "pass@4": 0.9997245203123646,
+            "pass@8": 0.9999788364839716,
+            "pass@16": 0.9999999148409177,
+            "pass@32": 0.9999999999998113,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9962367957746479,
+            "pass@2": 0.9972319646223802,
+            "pass@4": 0.9975970072402921,
+            "pass@8": 0.9979708642982112,
+            "pass@16": 0.9985716335094632,
+            "pass@32": 0.9993491975474896,
+            "pass@64": 0.9999188200066541,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9682852337110481,
+            "pass@2": 0.977452822823493,
+            "pass@4": 0.9842284054082472,
+            "pass@8": 0.9891870968251745,
+            "pass@16": 0.9931120473781875,
+            "pass@32": 0.9965309204049758,
+            "pass@64": 0.9991380576749798,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9647775423728814,
+            "pass@2": 0.9793001245607014,
+            "pass@4": 0.9885038096403199,
+            "pass@8": 0.9930489687895803,
+            "pass@16": 0.9957842683111874,
+            "pass@32": 0.9978504659297166,
+            "pass@64": 0.9992600866784821,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9408797021943573,
+            "pass@2": 0.9649356379433759,
+            "pass@4": 0.9808571087704949,
+            "pass@8": 0.9881229406714634,
+            "pass@16": 0.99055864455524,
+            "pass@32": 0.9913680560096522,
+            "pass@64": 0.9921630087220765,
+            "pass@128": 0.9937304075235109
+          },
+          "14": {
+            "pass@1": 0.9360640576323987,
+            "pass@2": 0.9562122856722352,
+            "pass@4": 0.9707606814919159,
+            "pass@8": 0.9804801905602375,
+            "pass@16": 0.9866715910082771,
+            "pass@32": 0.9909012287981317,
+            "pass@64": 0.9945214531172304,
+            "pass@128": 0.9968847352024922
+          },
+          "15": {
+            "pass@1": 0.9660404624277457,
+            "pass@2": 0.9819844198033775,
+            "pass@4": 0.989337811097312,
+            "pass@8": 0.9927511350152156,
+            "pass@16": 0.9945051007958996,
+            "pass@32": 0.9954700452049894,
+            "pass@64": 0.996392965878996,
+            "pass@128": 0.9971098265895953
+          },
+          "16": {
+            "pass@1": 0.8782762096774194,
+            "pass@2": 0.9086745142240289,
+            "pass@4": 0.9307146733674418,
+            "pass@8": 0.9456327971297461,
+            "pass@16": 0.9552193362938507,
+            "pass@32": 0.9633343766463572,
+            "pass@64": 0.9715414297733572,
+            "pass@128": 0.9806451612903225
+          },
+          "17": {
+            "pass@1": 0.8934248554913294,
+            "pass@2": 0.9237297175822676,
+            "pass@4": 0.9422482222236664,
+            "pass@8": 0.9540871391786083,
+            "pass@16": 0.9620315216397886,
+            "pass@32": 0.9683743703109818,
+            "pass@64": 0.9725996832538545,
+            "pass@128": 0.9739884393063584
+          },
+          "18": {
+            "pass@1": 0.8005357142857142,
+            "pass@2": 0.8531492547806522,
+            "pass@4": 0.8860468795864797,
+            "pass@8": 0.9091291155842222,
+            "pass@16": 0.9275554499538204,
+            "pass@32": 0.9442031158786668,
+            "pass@64": 0.9575396478067062,
+            "pass@128": 0.9657142857142857
+          },
+          "19": {
+            "pass@1": 0.6647428678678678,
+            "pass@2": 0.7474041462722564,
+            "pass@4": 0.8052284390689899,
+            "pass@8": 0.8453281707689558,
+            "pass@16": 0.8732022792173314,
+            "pass@32": 0.8947298964394246,
+            "pass@64": 0.9117678285429259,
+            "pass@128": 0.924924924924925
+          },
+          "20": {
+            "pass@1": 0.5938203828828829,
+            "pass@2": 0.6811821664183864,
+            "pass@4": 0.7462476041846112,
+            "pass@8": 0.7985336624704658,
+            "pass@16": 0.8402692553045812,
+            "pass@32": 0.8729885399962004,
+            "pass@64": 0.8982468595231101,
+            "pass@128": 0.918918918918919
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 821120,
+        "correct": 765697,
+        "answer_accuracy": 0.9325031664068589,
+        "avg_response_len": 291.1401585639127,
+        "resp_tokens_sum": 239061007,
+        "pass_at_k": {
+          "pass@1": 0.9325031664068589,
+          "pass@2": 0.9518363817271153,
+          "pass@4": 0.9646200336464147,
+          "pass@8": 0.9733466730553039,
+          "pass@16": 0.9793728174194902,
+          "pass@32": 0.983754479786111,
+          "pass@64": 0.9872180069196503,
+          "pass@128": 0.9900233826968043
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9433713573619632,
+            "pass@2": 0.9535782540698516,
+            "pass@4": 0.9618546829460732,
+            "pass@8": 0.968359742151891,
+            "pass@16": 0.9741511694787096,
+            "pass@32": 0.9805862212217205,
+            "pass@64": 0.9881224160445984,
+            "pass@128": 0.9938650306748467
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9983901515151515,
+            "pass@2": 0.999460898353615,
+            "pass@4": 0.9999225821204167,
+            "pass@8": 0.9999984049427112,
+            "pass@16": 0.9999999996635676,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.999791974852071,
+            "pass@2": 0.9999868960536739,
+            "pass@4": 0.9999999650561431,
+            "pass@8": 0.9999999999999813,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9935914855072464,
+            "pass@2": 0.9972134257674314,
+            "pass@4": 0.9991986126734157,
+            "pass@8": 0.9998860811105759,
+            "pass@16": 0.9999969943734291,
+            "pass@32": 0.9999999992176973,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999328438395415,
+            "pass@2": 0.99999894242267,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9981484826589595,
+            "pass@2": 0.9991423353511447,
+            "pass@4": 0.9997656553768929,
+            "pass@8": 0.9999830971478822,
+            "pass@16": 0.9999999407613452,
+            "pass@32": 0.9999999999999076,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9978213028169014,
+            "pass@2": 0.998696212709327,
+            "pass@4": 0.9994122448602376,
+            "pass@8": 0.9998848700630322,
+            "pass@16": 0.9999964254002791,
+            "pass@32": 0.9999999987973975,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9976609281437125,
+            "pass@2": 0.9988149930454053,
+            "pass@4": 0.9995776498184733,
+            "pass@8": 0.9999453661202221,
+            "pass@16": 0.9999993082440582,
+            "pass@32": 0.9999999999733661,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9791274472891566,
+            "pass@2": 0.9844943257281095,
+            "pass@4": 0.9885222114554959,
+            "pass@8": 0.9915842526776513,
+            "pass@16": 0.9936573596958077,
+            "pass@32": 0.9946784629745986,
+            "pass@64": 0.9954818436151764,
+            "pass@128": 0.9969879518072289
+          },
+          "12": {
+            "pass@1": 0.9556720219435737,
+            "pass@2": 0.9739207939426853,
+            "pass@4": 0.9861632180381843,
+            "pass@8": 0.9932283805113136,
+            "pass@16": 0.9967175800031935,
+            "pass@32": 0.9986257066354457,
+            "pass@64": 0.9997926928260641,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9692752100840336,
+            "pass@2": 0.9822620977524867,
+            "pass@4": 0.9899195350056031,
+            "pass@8": 0.9950258375460176,
+            "pass@16": 0.9984359616586164,
+            "pass@32": 0.9998247453476256,
+            "pass@64": 0.9999989636921891,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9544956140350878,
+            "pass@2": 0.9714041706957683,
+            "pass@4": 0.9850686652107085,
+            "pass@8": 0.9941410138686707,
+            "pass@16": 0.9982549357238066,
+            "pass@32": 0.9997482243275714,
+            "pass@64": 0.9999976075223646,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.962405303030303,
+            "pass@2": 0.9801527827487472,
+            "pass@4": 0.9897256777561895,
+            "pass@8": 0.9949554819709671,
+            "pass@16": 0.9975806792718727,
+            "pass@32": 0.9989957428151091,
+            "pass@64": 0.999819329898887,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.9157540954415955,
+            "pass@2": 0.9383107460125177,
+            "pass@4": 0.9528681665860143,
+            "pass@8": 0.9618255529076115,
+            "pass@16": 0.9666002487818389,
+            "pass@32": 0.9691287175972142,
+            "pass@64": 0.9713401996282022,
+            "pass@128": 0.9743589743589743
+          },
+          "17": {
+            "pass@1": 0.8866459627329193,
+            "pass@2": 0.9140307869125057,
+            "pass@4": 0.9320510835796145,
+            "pass@8": 0.9470251241563546,
+            "pass@16": 0.958828933724438,
+            "pass@32": 0.9669829599693496,
+            "pass@64": 0.9726542282720683,
+            "pass@128": 0.9782608695652174
+          },
+          "18": {
+            "pass@1": 0.8592830882352941,
+            "pass@2": 0.9078276980083376,
+            "pass@4": 0.9323671411477975,
+            "pass@8": 0.9477004466579106,
+            "pass@16": 0.9584144161429146,
+            "pass@32": 0.9654229587125491,
+            "pass@64": 0.9701825020374101,
+            "pass@128": 0.9735294117647059
+          },
+          "19": {
+            "pass@1": 0.6903366815476191,
+            "pass@2": 0.7712544232752157,
+            "pass@4": 0.8257569664841,
+            "pass@8": 0.8643543757323078,
+            "pass@16": 0.8929841094671906,
+            "pass@32": 0.914312749232494,
+            "pass@64": 0.9313348502509535,
+            "pass@128": 0.9464285714285714
+          },
+          "20": {
+            "pass@1": 0.6061938622754491,
+            "pass@2": 0.7069592360554483,
+            "pass@4": 0.779909319718268,
+            "pass@8": 0.8312956919046384,
+            "pass@16": 0.8691291374054729,
+            "pass@32": 0.9006392983887458,
+            "pass@64": 0.926877074969234,
+            "pass@128": 0.9461077844311377
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 801152,
+        "correct": 745158,
+        "answer_accuracy": 0.930108144272248,
+        "avg_response_len": 311.89763365753316,
+        "resp_tokens_sum": 249877413,
+        "pass_at_k": {
+          "pass@1": 0.930108144272248,
+          "pass@2": 0.9506625813159659,
+          "pass@4": 0.9639841486302464,
+          "pass@8": 0.9728187372779841,
+          "pass@16": 0.9789563417603896,
+          "pass@32": 0.9835196674940303,
+          "pass@64": 0.9868039392795732,
+          "pass@128": 0.9892954146029718
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9640391791044776,
+            "pass@2": 0.9751006287460335,
+            "pass@4": 0.9840680005484389,
+            "pass@8": 0.989525217666148,
+            "pass@16": 0.9922932987785924,
+            "pass@32": 0.9943696315299635,
+            "pass@64": 0.9970265954859788,
+            "pass@128": 1.0
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.9998857821637427,
+            "pass@2": 0.9999978415527007,
+            "pass@4": 0.9999999997259115,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9920266544117647,
+            "pass@2": 0.9942518961324689,
+            "pass@4": 0.9955414982134585,
+            "pass@8": 0.9967116901009395,
+            "pass@16": 0.998155080969405,
+            "pass@32": 0.9994198334387632,
+            "pass@64": 0.9999589349376307,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999260646687698,
+            "pass@2": 0.9999988356640751,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9976513053797469,
+            "pass@2": 0.9984395245689226,
+            "pass@4": 0.9992420280930073,
+            "pass@8": 0.9998284242652762,
+            "pass@16": 0.9999927222555876,
+            "pass@32": 0.9999999948561573,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9929147897897898,
+            "pass@2": 0.9970771706746117,
+            "pass@4": 0.9993902071362702,
+            "pass@8": 0.9999651367971397,
+            "pass@16": 0.9999998583663093,
+            "pass@32": 0.9999999999994196,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9923884646302251,
+            "pass@2": 0.9939457427146365,
+            "pass@4": 0.994643188391644,
+            "pass@8": 0.995451850487162,
+            "pass@16": 0.9964930745316808,
+            "pass@32": 0.9974718962737498,
+            "pass@64": 0.9983913004056533,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9703125,
+            "pass@2": 0.9826138920134985,
+            "pass@4": 0.9905110777819439,
+            "pass@8": 0.9952361673715535,
+            "pass@16": 0.9982486278795625,
+            "pass@32": 0.9997469296363813,
+            "pass@64": 0.9999980432330696,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9693950688073395,
+            "pass@2": 0.9864906149919334,
+            "pass@4": 0.9964843422668495,
+            "pass@8": 0.999587940904666,
+            "pass@16": 0.9999840639525752,
+            "pass@32": 0.9999999728189605,
+            "pass@64": 0.9999999999999997,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9416232638888888,
+            "pass@2": 0.9657007934772044,
+            "pass@4": 0.9807125428765847,
+            "pass@8": 0.9889479333237797,
+            "pass@16": 0.9935783281704044,
+            "pass@32": 0.9965795260843454,
+            "pass@64": 0.9983950138936485,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9647626112759644,
+            "pass@2": 0.9797271840463564,
+            "pass@4": 0.988019856104782,
+            "pass@8": 0.992716483465879,
+            "pass@16": 0.9955701659139958,
+            "pass@32": 0.9973444053304108,
+            "pass@64": 0.9984967309084293,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.9497010030864198,
+            "pass@2": 0.9660535597112858,
+            "pass@4": 0.9750617191369597,
+            "pass@8": 0.9813197913201593,
+            "pass@16": 0.9856122764931645,
+            "pass@32": 0.9887173282223192,
+            "pass@64": 0.991433089630488,
+            "pass@128": 0.9938271604938271
+          },
+          "16": {
+            "pass@1": 0.8941970870206489,
+            "pass@2": 0.9212493177014375,
+            "pass@4": 0.9393702139555561,
+            "pass@8": 0.9519523737165061,
+            "pass@16": 0.9612349386592156,
+            "pass@32": 0.9691753292432682,
+            "pass@64": 0.9747878703756769,
+            "pass@128": 0.976401179941003
+          },
+          "17": {
+            "pass@1": 0.8594455948795181,
+            "pass@2": 0.897720433900958,
+            "pass@4": 0.922174053073938,
+            "pass@8": 0.9382293341864976,
+            "pass@16": 0.9493378639982005,
+            "pass@32": 0.959124887388815,
+            "pass@64": 0.9681289020289342,
+            "pass@128": 0.9759036144578314
+          },
+          "18": {
+            "pass@1": 0.8485887096774194,
+            "pass@2": 0.903131746888494,
+            "pass@4": 0.9370043603420539,
+            "pass@8": 0.9578532946585311,
+            "pass@16": 0.9702169368171149,
+            "pass@32": 0.9777537348282815,
+            "pass@64": 0.9826360654372754,
+            "pass@128": 0.9870967741935484
+          },
+          "19": {
+            "pass@1": 0.7076331193353474,
+            "pass@2": 0.7887924625924787,
+            "pass@4": 0.8411277290300948,
+            "pass@8": 0.878659443062788,
+            "pass@16": 0.9053831900865986,
+            "pass@32": 0.9237568606242907,
+            "pass@64": 0.9352225250892545,
+            "pass@128": 0.9425981873111783
+          },
+          "20": {
+            "pass@1": 0.626196509009009,
+            "pass@2": 0.7125315522215124,
+            "pass@4": 0.7734252413268156,
+            "pass@8": 0.8190036582565382,
+            "pass@16": 0.8555370910100033,
+            "pass@32": 0.8847167374444287,
+            "pass@64": 0.9059201413232556,
+            "pass@128": 0.9219219219219219
+          }
+        }
+      }
+    }
+  }
+}

Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72a5925978851386faf56119b57768e1caa74f21607b7ec7511375d93471a972
+size 373196229

Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e985305d3fdfee25237f92c09828b1109fb641a1a34130bd6648ed3e186799e1
+size 422225883

Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/composition_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-midtrain-diff_ar-block4/composition", "results_dir": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs", "per_file": [{"file": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024488, "value_mismatch": 8666, "dependency_mismatch": 126198, "answer_mismatch": 7955, "extra_nodes": 11048, "missing_nodes": 8581, "perfect_with_extra": 938, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337887, "dependency_mismatch": 40758, "extra_nodes": 3484, "missing_nodes": 2571, "value_mismatch": 2633, "answer_mismatch": 2351, "perfect_with_extra": 420}, "teachers_in_school": {"total": 390656, "perfect": 346858, "value_mismatch": 3143, "answer_mismatch": 3007, "dependency_mismatch": 43394, "extra_nodes": 3920, "missing_nodes": 3330, "perfect_with_extra": 326}, "movie_festival_awards": {"total": 382336, "perfect": 339743, "dependency_mismatch": 42046, "extra_nodes": 3644, "missing_nodes": 2680, "perfect_with_extra": 192, "value_mismatch": 2890, "answer_mismatch": 2597}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/checkpoint-38646_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 645759, "value_mismatch": 214993, "dependency_mismatch": 610706, "answer_mismatch": 165476, "extra_nodes": 40541, "missing_nodes": 147475, "perfect_with_extra": 294, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 215781, "value_mismatch": 75701, "dependency_mismatch": 207056, "answer_mismatch": 59663, "extra_nodes": 15260, "perfect_with_extra": 59, "missing_nodes": 55087}, "movie_festival_awards": {"total": 418816, "perfect": 212276, "value_mismatch": 70597, "dependency_mismatch": 198554, "answer_mismatch": 53397, "missing_nodes": 45668, "extra_nodes": 12686, "perfect_with_extra": 95}, "teachers_in_school": {"total": 430464, "perfect": 217702, "value_mismatch": 68695, "dependency_mismatch": 205096, "answer_mismatch": 52416, "extra_nodes": 12595, "perfect_with_extra": 140, "missing_nodes": 46720}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/checkpoint-38646_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1670247, "value_mismatch": 223659, "dependency_mismatch": 736904, "answer_mismatch": 173431, "extra_nodes": 51589, "missing_nodes": 156056, "perfect_with_extra": 1232, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 553668, "dependency_mismatch": 247814, "extra_nodes": 18744, "missing_nodes": 57658, "value_mismatch": 78334, "answer_mismatch": 62014, "perfect_with_extra": 479}, "teachers_in_school": {"total": 821120, "perfect": 564560, "value_mismatch": 71838, "answer_mismatch": 55423, "dependency_mismatch": 248490, "extra_nodes": 16515, "missing_nodes": 50050, "perfect_with_extra": 466}, "movie_festival_awards": {"total": 801152, "perfect": 552019, "dependency_mismatch": 240600, "extra_nodes": 16330, "missing_nodes": 48348, "perfect_with_extra": 287, "value_mismatch": 73487, "answer_mismatch": 55994}}}}

Qwen3-100M-midtrain-diff_ar-block4/composition/graphs/results.json

@@ -0,0 +1,1780 @@
+{
+  "files": {
+    "checkpoint-38646_id_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "crazy_zootopia": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "3": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "4": {
+              "pass@1": 0.99375,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "5": {
+              "pass@1": 0.9587301587301588,
+              "pass@2": 0.9809523809523809,
+              "pass@4": 0.9936507936507937,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "6": {
+              "pass@1": 0.9550898203592815,
+              "pass@2": 0.9640718562874252,
+              "pass@4": 0.9940119760479041,
+              "pass@8": 0.9940119760479041,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "7": {
+              "pass@1": 0.893491124260355,
+              "pass@2": 0.9230769230769231,
+              "pass@4": 0.9644970414201184,
+              "pass@8": 0.985207100591716,
+              "pass@16": 0.9940828402366864,
+              "pass@32": 0.9970414201183432,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "8": {
+              "pass@1": 0.8301282051282052,
+              "pass@2": 0.8782051282051282,
+              "pass@4": 0.9423076923076923,
+              "pass@8": 0.9807692307692307,
+              "pass@16": 0.9935897435897436,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "9": {
+              "pass@1": 0.7633802816901408,
+              "pass@2": 0.8338028169014085,
+              "pass@4": 0.9070422535211268,
+              "pass@8": 0.9690140845070423,
+              "pass@16": 0.9887323943661972,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "10": {
+              "pass@1": 0.6371681415929203,
+              "pass@2": 0.7109144542772862,
+              "pass@4": 0.7905604719764012,
+              "pass@8": 0.8584070796460177,
+              "pass@16": 0.9056047197640118,
+              "pass@32": 0.9410029498525073,
+              "pass@64": 0.9557522123893806,
+              "pass@128": 0.9616519174041298
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.99375,
+              "5": 0.9587301587301588,
+              "6": 0.9550898203592815,
+              "7": 0.893491124260355,
+              "8": 0.8301282051282052,
+              "9": 0.7633802816901408,
+              "10": 0.6371681415929203
+            },
+            "pass@2": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9809523809523809,
+              "6": 0.9640718562874252,
+              "7": 0.9230769230769231,
+              "8": 0.8782051282051282,
+              "9": 0.8338028169014085,
+              "10": 0.7109144542772862
+            },
+            "pass@4": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9936507936507937,
+              "6": 0.9940119760479041,
+              "7": 0.9644970414201184,
+              "8": 0.9423076923076923,
+              "9": 0.9070422535211268,
+              "10": 0.7905604719764012
+            },
+            "pass@8": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 0.9940119760479041,
+              "7": 0.985207100591716,
+              "8": 0.9807692307692307,
+              "9": 0.9690140845070423,
+              "10": 0.8584070796460177
+            },
+            "pass@16": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 0.9940828402366864,
+              "8": 0.9935897435897436,
+              "9": 0.9887323943661972,
+              "10": 0.9056047197640118
+            },
+            "pass@32": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 0.9970414201183432,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9410029498525073
+            },
+            "pass@64": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9557522123893806
+            },
+            "pass@128": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9616519174041298
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.8899020601148261,
+            "pass@2": 0.9192840256670044,
+            "pass@4": 0.9533941236068896,
+            "pass@8": 0.9756838905775076,
+            "pass@16": 0.9864910503208375,
+            "pass@32": 0.9929078014184397,
+            "pass@64": 0.9949341438703141,
+            "pass@128": 0.9956095913542722
+          }
+        },
+        "movie_festival_awards": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "3": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "4": {
+              "pass@1": 0.9883040935672515,
+              "pass@2": 0.9941520467836257,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "5": {
+              "pass@1": 0.95,
+              "pass@2": 0.9735294117647059,
+              "pass@4": 0.9911764705882353,
+              "pass@8": 0.9941176470588236,
+              "pass@16": 0.9970588235294118,
+              "pass@32": 0.9970588235294118,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "6": {
+              "pass@1": 0.9684542586750788,
+              "pass@2": 0.9810725552050473,
+              "pass@4": 0.9842271293375394,
+              "pass@8": 0.9936908517350158,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "7": {
+              "pass@1": 0.8575949367088608,
+              "pass@2": 0.9145569620253164,
+              "pass@4": 0.9651898734177216,
+              "pass@8": 0.9936708860759493,
+              "pass@16": 0.9968354430379747,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "8": {
+              "pass@1": 0.8168168168168168,
+              "pass@2": 0.8708708708708709,
+              "pass@4": 0.9129129129129129,
+              "pass@8": 0.960960960960961,
+              "pass@16": 0.993993993993994,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "9": {
+              "pass@1": 0.7652733118971061,
+              "pass@2": 0.8263665594855305,
+              "pass@4": 0.9035369774919614,
+              "pass@8": 0.9453376205787781,
+              "pass@16": 0.9646302250803859,
+              "pass@32": 0.9935691318327974,
+              "pass@64": 0.9967845659163987,
+              "pass@128": 0.9967845659163987
+            },
+            "10": {
+              "pass@1": 0.6149253731343284,
+              "pass@2": 0.6895522388059702,
+              "pass@4": 0.7701492537313432,
+              "pass@8": 0.8716417910447761,
+              "pass@16": 0.9343283582089552,
+              "pass@32": 0.9552238805970149,
+              "pass@64": 0.9791044776119403,
+              "pass@128": 0.9880597014925373
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9883040935672515,
+              "5": 0.95,
+              "6": 0.9684542586750788,
+              "7": 0.8575949367088608,
+              "8": 0.8168168168168168,
+              "9": 0.7652733118971061,
+              "10": 0.6149253731343284
+            },
+            "pass@2": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9941520467836257,
+              "5": 0.9735294117647059,
+              "6": 0.9810725552050473,
+              "7": 0.9145569620253164,
+              "8": 0.8708708708708709,
+              "9": 0.8263665594855305,
+              "10": 0.6895522388059702
+            },
+            "pass@4": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9911764705882353,
+              "6": 0.9842271293375394,
+              "7": 0.9651898734177216,
+              "8": 0.9129129129129129,
+              "9": 0.9035369774919614,
+              "10": 0.7701492537313432
+            },
+            "pass@8": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9941176470588236,
+              "6": 0.9936908517350158,
+              "7": 0.9936708860759493,
+              "8": 0.960960960960961,
+              "9": 0.9453376205787781,
+              "10": 0.8716417910447761
+            },
+            "pass@16": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9970588235294118,
+              "6": 1.0,
+              "7": 0.9968354430379747,
+              "8": 0.993993993993994,
+              "9": 0.9646302250803859,
+              "10": 0.9343283582089552
+            },
+            "pass@32": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9970588235294118,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 0.9935691318327974,
+              "10": 0.9552238805970149
+            },
+            "pass@64": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 0.9967845659163987,
+              "10": 0.9791044776119403
+            },
+            "pass@128": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 0.9967845659163987,
+              "10": 0.9880597014925373
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.8865082022095748,
+            "pass@2": 0.9179779042517576,
+            "pass@4": 0.9481084700368263,
+            "pass@8": 0.9735520589219954,
+            "pass@16": 0.987612989621694,
+            "pass@32": 0.9939738868429863,
+            "pass@64": 0.9973217274857716,
+            "pass@128": 0.9983260796786073
+          }
+        },
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "2": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "3": {
+              "pass@1": 1.0,
+              "pass@2": 1.0,
+              "pass@4": 1.0,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "4": {
+              "pass@1": 0.985207100591716,
+              "pass@2": 0.9881656804733728,
+              "pass@4": 0.9970414201183432,
+              "pass@8": 1.0,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "5": {
+              "pass@1": 0.9478260869565217,
+              "pass@2": 0.9826086956521739,
+              "pass@4": 0.991304347826087,
+              "pass@8": 0.9971014492753624,
+              "pass@16": 1.0,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "6": {
+              "pass@1": 0.9770773638968482,
+              "pass@2": 0.9856733524355301,
+              "pass@4": 0.9914040114613181,
+              "pass@8": 0.997134670487106,
+              "pass@16": 0.997134670487106,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "7": {
+              "pass@1": 0.8554913294797688,
+              "pass@2": 0.9017341040462428,
+              "pass@4": 0.9595375722543352,
+              "pass@8": 0.9826589595375722,
+              "pass@16": 0.9913294797687862,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "8": {
+              "pass@1": 0.819718309859155,
+              "pass@2": 0.8788732394366198,
+              "pass@4": 0.952112676056338,
+              "pass@8": 0.9830985915492958,
+              "pass@16": 0.9971830985915493,
+              "pass@32": 1.0,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "9": {
+              "pass@1": 0.7455089820359282,
+              "pass@2": 0.8023952095808383,
+              "pass@4": 0.8952095808383234,
+              "pass@8": 0.9580838323353293,
+              "pass@16": 0.9850299401197605,
+              "pass@32": 0.9910179640718563,
+              "pass@64": 1.0,
+              "pass@128": 1.0
+            },
+            "10": {
+              "pass@1": 0.6319018404907976,
+              "pass@2": 0.6932515337423313,
+              "pass@4": 0.7668711656441718,
+              "pass@8": 0.8312883435582822,
+              "pass@16": 0.8957055214723927,
+              "pass@32": 0.9478527607361963,
+              "pass@64": 0.9570552147239264,
+              "pass@128": 0.9693251533742331
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.985207100591716,
+              "5": 0.9478260869565217,
+              "6": 0.9770773638968482,
+              "7": 0.8554913294797688,
+              "8": 0.819718309859155,
+              "9": 0.7455089820359282,
+              "10": 0.6319018404907976
+            },
+            "pass@2": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9881656804733728,
+              "5": 0.9826086956521739,
+              "6": 0.9856733524355301,
+              "7": 0.9017341040462428,
+              "8": 0.8788732394366198,
+              "9": 0.8023952095808383,
+              "10": 0.6932515337423313
+            },
+            "pass@4": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 0.9970414201183432,
+              "5": 0.991304347826087,
+              "6": 0.9914040114613181,
+              "7": 0.9595375722543352,
+              "8": 0.952112676056338,
+              "9": 0.8952095808383234,
+              "10": 0.7668711656441718
+            },
+            "pass@8": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 0.9971014492753624,
+              "6": 0.997134670487106,
+              "7": 0.9826589595375722,
+              "8": 0.9830985915492958,
+              "9": 0.9580838323353293,
+              "10": 0.8312883435582822
+            },
+            "pass@16": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 0.997134670487106,
+              "7": 0.9913294797687862,
+              "8": 0.9971830985915493,
+              "9": 0.9850299401197605,
+              "10": 0.8957055214723927
+            },
+            "pass@32": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 0.9910179640718563,
+              "10": 0.9478527607361963
+            },
+            "pass@64": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9570552147239264
+            },
+            "pass@128": {
+              "2": 1.0,
+              "3": 1.0,
+              "4": 1.0,
+              "5": 1.0,
+              "6": 1.0,
+              "7": 1.0,
+              "8": 1.0,
+              "9": 1.0,
+              "10": 0.9693251533742331
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.8853211009174312,
+            "pass@2": 0.9154652686762779,
+            "pass@4": 0.9511795543905636,
+            "pass@8": 0.9728047182175622,
+            "pass@16": 0.9855832241153342,
+            "pass@32": 0.9934469200524246,
+            "pass@64": 0.9954128440366973,
+            "pass@128": 0.9967234600262124
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.989,
+            "pass@2": 0.994,
+            "pass@4": 0.999,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.952,
+            "pass@2": 0.979,
+            "pass@4": 0.992,
+            "pass@8": 0.997,
+            "pass@16": 0.999,
+            "pass@32": 0.999,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.967,
+            "pass@2": 0.977,
+            "pass@4": 0.99,
+            "pass@8": 0.995,
+            "pass@16": 0.999,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.869,
+            "pass@2": 0.913,
+            "pass@4": 0.963,
+            "pass@8": 0.987,
+            "pass@16": 0.994,
+            "pass@32": 0.999,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.822,
+            "pass@2": 0.876,
+            "pass@4": 0.936,
+            "pass@8": 0.975,
+            "pass@16": 0.995,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.758,
+            "pass@2": 0.821,
+            "pass@4": 0.902,
+            "pass@8": 0.958,
+            "pass@16": 0.98,
+            "pass@32": 0.995,
+            "pass@64": 0.999,
+            "pass@128": 0.999
+          },
+          "10": {
+            "pass@1": 0.628,
+            "pass@2": 0.698,
+            "pass@4": 0.776,
+            "pass@8": 0.854,
+            "pass@16": 0.912,
+            "pass@32": 0.948,
+            "pass@64": 0.964,
+            "pass@128": 0.973
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 0.989,
+            "5": 0.952,
+            "6": 0.967,
+            "7": 0.869,
+            "8": 0.822,
+            "9": 0.758,
+            "10": 0.628
+          },
+          "pass@2": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 0.994,
+            "5": 0.979,
+            "6": 0.977,
+            "7": 0.913,
+            "8": 0.876,
+            "9": 0.821,
+            "10": 0.698
+          },
+          "pass@4": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 0.999,
+            "5": 0.992,
+            "6": 0.99,
+            "7": 0.963,
+            "8": 0.936,
+            "9": 0.902,
+            "10": 0.776
+          },
+          "pass@8": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.997,
+            "6": 0.995,
+            "7": 0.987,
+            "8": 0.975,
+            "9": 0.958,
+            "10": 0.854
+          },
+          "pass@16": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.999,
+            "6": 0.999,
+            "7": 0.994,
+            "8": 0.995,
+            "9": 0.98,
+            "10": 0.912
+          },
+          "pass@32": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 0.999,
+            "6": 1.0,
+            "7": 0.999,
+            "8": 1.0,
+            "9": 0.995,
+            "10": 0.948
+          },
+          "pass@64": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 1.0,
+            "6": 1.0,
+            "7": 1.0,
+            "8": 1.0,
+            "9": 0.999,
+            "10": 0.964
+          },
+          "pass@128": {
+            "2": 1.0,
+            "3": 1.0,
+            "4": 1.0,
+            "5": 1.0,
+            "6": 1.0,
+            "7": 1.0,
+            "8": 1.0,
+            "9": 0.999,
+            "10": 0.973
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.8872222222222222,
+          "pass@2": 0.9175555555555556,
+          "pass@4": 0.9508888888888889,
+          "pass@8": 0.974,
+          "pass@16": 0.9865555555555555,
+          "pass@32": 0.9934444444444445,
+          "pass@64": 0.9958888888888889,
+          "pass@128": 0.9968888888888889
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "crazy_zootopia": {
+            "per_op": {
+              "2": 321,
+              "3": 327,
+              "4": 320,
+              "5": 315,
+              "6": 334,
+              "7": 338,
+              "8": 312,
+              "9": 355,
+              "10": 339
+            },
+            "total": 2961
+          },
+          "movie_festival_awards": {
+            "per_op": {
+              "2": 350,
+              "3": 343,
+              "4": 342,
+              "5": 340,
+              "6": 317,
+              "7": 316,
+              "8": 333,
+              "9": 311,
+              "10": 335
+            },
+            "total": 2987
+          },
+          "teachers_in_school": {
+            "per_op": {
+              "2": 329,
+              "3": 330,
+              "4": 338,
+              "5": 345,
+              "6": 349,
+              "7": 346,
+              "8": 355,
+              "9": 334,
+              "10": 326
+            },
+            "total": 3052
+          }
+        },
+        "total": {
+          "per_op": {
+            "2": 1000,
+            "3": 1000,
+            "4": 1000,
+            "5": 1000,
+            "6": 1000,
+            "7": 1000,
+            "8": 1000,
+            "9": 1000,
+            "10": 1000
+          },
+          "total": 9000
+        }
+      }
+    },
+    "checkpoint-38646_ood_generations.jsonl.graph_results.jsonl": {
+      "per_template": {
+        "crazy_zootopia": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.5920679886685553,
+              "pass@2": 0.6798866855524079,
+              "pass@4": 0.7648725212464589,
+              "pass@8": 0.8470254957507082,
+              "pass@16": 0.9206798866855525,
+              "pass@32": 0.9603399433427762,
+              "pass@64": 0.9801699716713881,
+              "pass@128": 0.9858356940509915
+            },
+            "12": {
+              "pass@1": 0.6384180790960452,
+              "pass@2": 0.6836158192090396,
+              "pass@4": 0.768361581920904,
+              "pass@8": 0.867231638418079,
+              "pass@16": 0.923728813559322,
+              "pass@32": 0.9491525423728814,
+              "pass@64": 0.9717514124293786,
+              "pass@128": 0.9774011299435028
+            },
+            "13": {
+              "pass@1": 0.5862068965517241,
+              "pass@2": 0.6426332288401254,
+              "pass@4": 0.7241379310344828,
+              "pass@8": 0.8119122257053292,
+              "pass@16": 0.8746081504702194,
+              "pass@32": 0.9090909090909091,
+              "pass@64": 0.9373040752351097,
+              "pass@128": 0.9592476489028213
+            },
+            "14": {
+              "pass@1": 0.5514018691588785,
+              "pass@2": 0.5856697819314641,
+              "pass@4": 0.6261682242990654,
+              "pass@8": 0.6853582554517134,
+              "pass@16": 0.7476635514018691,
+              "pass@32": 0.8473520249221184,
+              "pass@64": 0.9003115264797508,
+              "pass@128": 0.9283489096573209
+            },
+            "15": {
+              "pass@1": 0.5028901734104047,
+              "pass@2": 0.5404624277456648,
+              "pass@4": 0.5867052023121387,
+              "pass@8": 0.6763005780346821,
+              "pass@16": 0.7485549132947977,
+              "pass@32": 0.838150289017341,
+              "pass@64": 0.9104046242774566,
+              "pass@128": 0.9508670520231214
+            },
+            "16": {
+              "pass@1": 0.4967741935483871,
+              "pass@2": 0.535483870967742,
+              "pass@4": 0.5838709677419355,
+              "pass@8": 0.6225806451612903,
+              "pass@16": 0.6838709677419355,
+              "pass@32": 0.7129032258064516,
+              "pass@64": 0.7645161290322581,
+              "pass@128": 0.7935483870967742
+            },
+            "17": {
+              "pass@1": 0.5433526011560693,
+              "pass@2": 0.5780346820809249,
+              "pass@4": 0.615606936416185,
+              "pass@8": 0.6445086705202312,
+              "pass@16": 0.6936416184971098,
+              "pass@32": 0.7312138728323699,
+              "pass@64": 0.7832369942196532,
+              "pass@128": 0.8352601156069365
+            },
+            "18": {
+              "pass@1": 0.43714285714285717,
+              "pass@2": 0.46,
+              "pass@4": 0.4828571428571429,
+              "pass@8": 0.5142857142857142,
+              "pass@16": 0.5742857142857143,
+              "pass@32": 0.6371428571428571,
+              "pass@64": 0.6971428571428572,
+              "pass@128": 0.7714285714285715
+            },
+            "19": {
+              "pass@1": 0.34234234234234234,
+              "pass@2": 0.4084084084084084,
+              "pass@4": 0.44144144144144143,
+              "pass@8": 0.4804804804804805,
+              "pass@16": 0.5225225225225225,
+              "pass@32": 0.5585585585585585,
+              "pass@64": 0.5975975975975976,
+              "pass@128": 0.6276276276276276
+            },
+            "20": {
+              "pass@1": 0.27627627627627627,
+              "pass@2": 0.3303303303303303,
+              "pass@4": 0.35135135135135137,
+              "pass@8": 0.4084084084084084,
+              "pass@16": 0.44744744744744747,
+              "pass@32": 0.5105105105105106,
+              "pass@64": 0.5705705705705706,
+              "pass@128": 0.6246246246246246
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.5920679886685553,
+              "12": 0.6384180790960452,
+              "13": 0.5862068965517241,
+              "14": 0.5514018691588785,
+              "15": 0.5028901734104047,
+              "16": 0.4967741935483871,
+              "17": 0.5433526011560693,
+              "18": 0.43714285714285717,
+              "19": 0.34234234234234234,
+              "20": 0.27627627627627627
+            },
+            "pass@2": {
+              "11": 0.6798866855524079,
+              "12": 0.6836158192090396,
+              "13": 0.6426332288401254,
+              "14": 0.5856697819314641,
+              "15": 0.5404624277456648,
+              "16": 0.535483870967742,
+              "17": 0.5780346820809249,
+              "18": 0.46,
+              "19": 0.4084084084084084,
+              "20": 0.3303303303303303
+            },
+            "pass@4": {
+              "11": 0.7648725212464589,
+              "12": 0.768361581920904,
+              "13": 0.7241379310344828,
+              "14": 0.6261682242990654,
+              "15": 0.5867052023121387,
+              "16": 0.5838709677419355,
+              "17": 0.615606936416185,
+              "18": 0.4828571428571429,
+              "19": 0.44144144144144143,
+              "20": 0.35135135135135137
+            },
+            "pass@8": {
+              "11": 0.8470254957507082,
+              "12": 0.867231638418079,
+              "13": 0.8119122257053292,
+              "14": 0.6853582554517134,
+              "15": 0.6763005780346821,
+              "16": 0.6225806451612903,
+              "17": 0.6445086705202312,
+              "18": 0.5142857142857142,
+              "19": 0.4804804804804805,
+              "20": 0.4084084084084084
+            },
+            "pass@16": {
+              "11": 0.9206798866855525,
+              "12": 0.923728813559322,
+              "13": 0.8746081504702194,
+              "14": 0.7476635514018691,
+              "15": 0.7485549132947977,
+              "16": 0.6838709677419355,
+              "17": 0.6936416184971098,
+              "18": 0.5742857142857143,
+              "19": 0.5225225225225225,
+              "20": 0.44744744744744747
+            },
+            "pass@32": {
+              "11": 0.9603399433427762,
+              "12": 0.9491525423728814,
+              "13": 0.9090909090909091,
+              "14": 0.8473520249221184,
+              "15": 0.838150289017341,
+              "16": 0.7129032258064516,
+              "17": 0.7312138728323699,
+              "18": 0.6371428571428571,
+              "19": 0.5585585585585585,
+              "20": 0.5105105105105106
+            },
+            "pass@64": {
+              "11": 0.9801699716713881,
+              "12": 0.9717514124293786,
+              "13": 0.9373040752351097,
+              "14": 0.9003115264797508,
+              "15": 0.9104046242774566,
+              "16": 0.7645161290322581,
+              "17": 0.7832369942196532,
+              "18": 0.6971428571428572,
+              "19": 0.5975975975975976,
+              "20": 0.5705705705705706
+            },
+            "pass@128": {
+              "11": 0.9858356940509915,
+              "12": 0.9774011299435028,
+              "13": 0.9592476489028213,
+              "14": 0.9283489096573209,
+              "15": 0.9508670520231214,
+              "16": 0.7935483870967742,
+              "17": 0.8352601156069365,
+              "18": 0.7714285714285715,
+              "19": 0.6276276276276276,
+              "20": 0.6246246246246246
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.4974739970282318,
+            "pass@2": 0.5453194650817236,
+            "pass@4": 0.5955423476968796,
+            "pass@8": 0.6570579494799406,
+            "pass@16": 0.7150074294205052,
+            "pass@32": 0.7667161961367014,
+            "pass@64": 0.812481426448737,
+            "pass@128": 0.8466567607726597
+          }
+        },
+        "movie_festival_awards": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.6,
+              "pass@2": 0.6666666666666666,
+              "pass@4": 0.7619047619047619,
+              "pass@8": 0.8698412698412699,
+              "pass@16": 0.926984126984127,
+              "pass@32": 0.9619047619047619,
+              "pass@64": 0.9682539682539683,
+              "pass@128": 0.9873015873015873
+            },
+            "12": {
+              "pass@1": 0.636085626911315,
+              "pass@2": 0.709480122324159,
+              "pass@4": 0.7981651376146789,
+              "pass@8": 0.8776758409785933,
+              "pass@16": 0.9296636085626911,
+              "pass@32": 0.963302752293578,
+              "pass@64": 0.9755351681957186,
+              "pass@128": 0.9877675840978594
+            },
+            "13": {
+              "pass@1": 0.5802469135802469,
+              "pass@2": 0.6820987654320988,
+              "pass@4": 0.7561728395061729,
+              "pass@8": 0.8240740740740741,
+              "pass@16": 0.8827160493827161,
+              "pass@32": 0.9320987654320988,
+              "pass@64": 0.9629629629629629,
+              "pass@128": 0.9722222222222222
+            },
+            "14": {
+              "pass@1": 0.5608308605341247,
+              "pass@2": 0.5964391691394659,
+              "pass@4": 0.6617210682492581,
+              "pass@8": 0.7388724035608308,
+              "pass@16": 0.8011869436201781,
+              "pass@32": 0.8545994065281899,
+              "pass@64": 0.9109792284866469,
+              "pass@128": 0.9376854599406528
+            },
+            "15": {
+              "pass@1": 0.5,
+              "pass@2": 0.5462962962962963,
+              "pass@4": 0.5895061728395061,
+              "pass@8": 0.6574074074074074,
+              "pass@16": 0.7253086419753086,
+              "pass@32": 0.7870370370370371,
+              "pass@64": 0.8765432098765432,
+              "pass@128": 0.9259259259259259
+            },
+            "16": {
+              "pass@1": 0.5368731563421829,
+              "pass@2": 0.584070796460177,
+              "pass@4": 0.6135693215339233,
+              "pass@8": 0.6666666666666666,
+              "pass@16": 0.7168141592920354,
+              "pass@32": 0.7699115044247787,
+              "pass@64": 0.8082595870206489,
+              "pass@128": 0.8348082595870207
+            },
+            "17": {
+              "pass@1": 0.4939759036144578,
+              "pass@2": 0.5120481927710844,
+              "pass@4": 0.536144578313253,
+              "pass@8": 0.5933734939759037,
+              "pass@16": 0.6566265060240963,
+              "pass@32": 0.7198795180722891,
+              "pass@64": 0.7740963855421686,
+              "pass@128": 0.8192771084337349
+            },
+            "18": {
+              "pass@1": 0.44516129032258067,
+              "pass@2": 0.5064516129032258,
+              "pass@4": 0.532258064516129,
+              "pass@8": 0.5645161290322581,
+              "pass@16": 0.6258064516129033,
+              "pass@32": 0.6774193548387096,
+              "pass@64": 0.7483870967741936,
+              "pass@128": 0.7935483870967742
+            },
+            "19": {
+              "pass@1": 0.3776435045317221,
+              "pass@2": 0.4501510574018127,
+              "pass@4": 0.4652567975830816,
+              "pass@8": 0.5105740181268882,
+              "pass@16": 0.5317220543806647,
+              "pass@32": 0.5800604229607251,
+              "pass@64": 0.622356495468278,
+              "pass@128": 0.6737160120845922
+            },
+            "20": {
+              "pass@1": 0.2912912912912913,
+              "pass@2": 0.34534534534534533,
+              "pass@4": 0.37537537537537535,
+              "pass@8": 0.4084084084084084,
+              "pass@16": 0.4624624624624625,
+              "pass@32": 0.5165165165165165,
+              "pass@64": 0.5645645645645646,
+              "pass@128": 0.6096096096096096
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.6,
+              "12": 0.636085626911315,
+              "13": 0.5802469135802469,
+              "14": 0.5608308605341247,
+              "15": 0.5,
+              "16": 0.5368731563421829,
+              "17": 0.4939759036144578,
+              "18": 0.44516129032258067,
+              "19": 0.3776435045317221,
+              "20": 0.2912912912912913
+            },
+            "pass@2": {
+              "11": 0.6666666666666666,
+              "12": 0.709480122324159,
+              "13": 0.6820987654320988,
+              "14": 0.5964391691394659,
+              "15": 0.5462962962962963,
+              "16": 0.584070796460177,
+              "17": 0.5120481927710844,
+              "18": 0.5064516129032258,
+              "19": 0.4501510574018127,
+              "20": 0.34534534534534533
+            },
+            "pass@4": {
+              "11": 0.7619047619047619,
+              "12": 0.7981651376146789,
+              "13": 0.7561728395061729,
+              "14": 0.6617210682492581,
+              "15": 0.5895061728395061,
+              "16": 0.6135693215339233,
+              "17": 0.536144578313253,
+              "18": 0.532258064516129,
+              "19": 0.4652567975830816,
+              "20": 0.37537537537537535
+            },
+            "pass@8": {
+              "11": 0.8698412698412699,
+              "12": 0.8776758409785933,
+              "13": 0.8240740740740741,
+              "14": 0.7388724035608308,
+              "15": 0.6574074074074074,
+              "16": 0.6666666666666666,
+              "17": 0.5933734939759037,
+              "18": 0.5645161290322581,
+              "19": 0.5105740181268882,
+              "20": 0.4084084084084084
+            },
+            "pass@16": {
+              "11": 0.926984126984127,
+              "12": 0.9296636085626911,
+              "13": 0.8827160493827161,
+              "14": 0.8011869436201781,
+              "15": 0.7253086419753086,
+              "16": 0.7168141592920354,
+              "17": 0.6566265060240963,
+              "18": 0.6258064516129033,
+              "19": 0.5317220543806647,
+              "20": 0.4624624624624625
+            },
+            "pass@32": {
+              "11": 0.9619047619047619,
+              "12": 0.963302752293578,
+              "13": 0.9320987654320988,
+              "14": 0.8545994065281899,
+              "15": 0.7870370370370371,
+              "16": 0.7699115044247787,
+              "17": 0.7198795180722891,
+              "18": 0.6774193548387096,
+              "19": 0.5800604229607251,
+              "20": 0.5165165165165165
+            },
+            "pass@64": {
+              "11": 0.9682539682539683,
+              "12": 0.9755351681957186,
+              "13": 0.9629629629629629,
+              "14": 0.9109792284866469,
+              "15": 0.8765432098765432,
+              "16": 0.8082595870206489,
+              "17": 0.7740963855421686,
+              "18": 0.7483870967741936,
+              "19": 0.622356495468278,
+              "20": 0.5645645645645646
+            },
+            "pass@128": {
+              "11": 0.9873015873015873,
+              "12": 0.9877675840978594,
+              "13": 0.9722222222222222,
+              "14": 0.9376854599406528,
+              "15": 0.9259259259259259,
+              "16": 0.8348082595870207,
+              "17": 0.8192771084337349,
+              "18": 0.7935483870967742,
+              "19": 0.6737160120845922,
+              "20": 0.6096096096096096
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.5018337408312958,
+            "pass@2": 0.5592909535452323,
+            "pass@4": 0.6081907090464548,
+            "pass@8": 0.6702322738386308,
+            "pass@16": 0.7249388753056235,
+            "pass@32": 0.7753667481662592,
+            "pass@64": 0.8202933985330073,
+            "pass@128": 0.8533007334963325
+          }
+        },
+        "teachers_in_school": {
+          "per_op_pass_at_k": {
+            "11": {
+              "pass@1": 0.6626506024096386,
+              "pass@2": 0.6987951807228916,
+              "pass@4": 0.7921686746987951,
+              "pass@8": 0.8524096385542169,
+              "pass@16": 0.927710843373494,
+              "pass@32": 0.963855421686747,
+              "pass@64": 0.9728915662650602,
+              "pass@128": 0.9789156626506024
+            },
+            "12": {
+              "pass@1": 0.5862068965517241,
+              "pass@2": 0.6551724137931034,
+              "pass@4": 0.7492163009404389,
+              "pass@8": 0.8401253918495298,
+              "pass@16": 0.9090909090909091,
+              "pass@32": 0.9561128526645768,
+              "pass@64": 0.9780564263322884,
+              "pass@128": 0.9811912225705329
+            },
+            "13": {
+              "pass@1": 0.6106442577030813,
+              "pass@2": 0.6694677871148459,
+              "pass@4": 0.7675070028011205,
+              "pass@8": 0.8711484593837535,
+              "pass@16": 0.927170868347339,
+              "pass@32": 0.9523809523809523,
+              "pass@64": 0.969187675070028,
+              "pass@128": 0.9915966386554622
+            },
+            "14": {
+              "pass@1": 0.5760233918128655,
+              "pass@2": 0.631578947368421,
+              "pass@4": 0.6900584795321637,
+              "pass@8": 0.7543859649122807,
+              "pass@16": 0.8099415204678363,
+              "pass@32": 0.8859649122807017,
+              "pass@64": 0.9502923976608187,
+              "pass@128": 0.9736842105263158
+            },
+            "15": {
+              "pass@1": 0.5484848484848485,
+              "pass@2": 0.5848484848484848,
+              "pass@4": 0.6090909090909091,
+              "pass@8": 0.6666666666666666,
+              "pass@16": 0.7333333333333333,
+              "pass@32": 0.8272727272727273,
+              "pass@64": 0.896969696969697,
+              "pass@128": 0.9454545454545454
+            },
+            "16": {
+              "pass@1": 0.5384615384615384,
+              "pass@2": 0.5641025641025641,
+              "pass@4": 0.5982905982905983,
+              "pass@8": 0.6524216524216524,
+              "pass@16": 0.6866096866096866,
+              "pass@32": 0.7606837606837606,
+              "pass@64": 0.8034188034188035,
+              "pass@128": 0.8376068376068376
+            },
+            "17": {
+              "pass@1": 0.4254658385093168,
+              "pass@2": 0.4472049689440994,
+              "pass@4": 0.5093167701863354,
+              "pass@8": 0.5683229813664596,
+              "pass@16": 0.6242236024844721,
+              "pass@32": 0.6956521739130435,
+              "pass@64": 0.7795031055900621,
+              "pass@128": 0.8136645962732919
+            },
+            "18": {
+              "pass@1": 0.4176470588235294,
+              "pass@2": 0.45,
+              "pass@4": 0.47941176470588237,
+              "pass@8": 0.5205882352941177,
+              "pass@16": 0.5647058823529412,
+              "pass@32": 0.5794117647058824,
+              "pass@64": 0.6823529411764706,
+              "pass@128": 0.7529411764705882
+            },
+            "19": {
+              "pass@1": 0.375,
+              "pass@2": 0.40773809523809523,
+              "pass@4": 0.44047619047619047,
+              "pass@8": 0.4851190476190476,
+              "pass@16": 0.5208333333333334,
+              "pass@32": 0.5744047619047619,
+              "pass@64": 0.6190476190476191,
+              "pass@128": 0.6755952380952381
+            },
+            "20": {
+              "pass@1": 0.2844311377245509,
+              "pass@2": 0.33532934131736525,
+              "pass@4": 0.38323353293413176,
+              "pass@8": 0.4281437125748503,
+              "pass@16": 0.47005988023952094,
+              "pass@32": 0.5209580838323353,
+              "pass@64": 0.5658682634730539,
+              "pass@128": 0.6287425149700598
+            }
+          },
+          "per_op_pass_at_columns": {
+            "pass@1": {
+              "11": 0.6626506024096386,
+              "12": 0.5862068965517241,
+              "13": 0.6106442577030813,
+              "14": 0.5760233918128655,
+              "15": 0.5484848484848485,
+              "16": 0.5384615384615384,
+              "17": 0.4254658385093168,
+              "18": 0.4176470588235294,
+              "19": 0.375,
+              "20": 0.2844311377245509
+            },
+            "pass@2": {
+              "11": 0.6987951807228916,
+              "12": 0.6551724137931034,
+              "13": 0.6694677871148459,
+              "14": 0.631578947368421,
+              "15": 0.5848484848484848,
+              "16": 0.5641025641025641,
+              "17": 0.4472049689440994,
+              "18": 0.45,
+              "19": 0.40773809523809523,
+              "20": 0.33532934131736525
+            },
+            "pass@4": {
+              "11": 0.7921686746987951,
+              "12": 0.7492163009404389,
+              "13": 0.7675070028011205,
+              "14": 0.6900584795321637,
+              "15": 0.6090909090909091,
+              "16": 0.5982905982905983,
+              "17": 0.5093167701863354,
+              "18": 0.47941176470588237,
+              "19": 0.44047619047619047,
+              "20": 0.38323353293413176
+            },
+            "pass@8": {
+              "11": 0.8524096385542169,
+              "12": 0.8401253918495298,
+              "13": 0.8711484593837535,
+              "14": 0.7543859649122807,
+              "15": 0.6666666666666666,
+              "16": 0.6524216524216524,
+              "17": 0.5683229813664596,
+              "18": 0.5205882352941177,
+              "19": 0.4851190476190476,
+              "20": 0.4281437125748503
+            },
+            "pass@16": {
+              "11": 0.927710843373494,
+              "12": 0.9090909090909091,
+              "13": 0.927170868347339,
+              "14": 0.8099415204678363,
+              "15": 0.7333333333333333,
+              "16": 0.6866096866096866,
+              "17": 0.6242236024844721,
+              "18": 0.5647058823529412,
+              "19": 0.5208333333333334,
+              "20": 0.47005988023952094
+            },
+            "pass@32": {
+              "11": 0.963855421686747,
+              "12": 0.9561128526645768,
+              "13": 0.9523809523809523,
+              "14": 0.8859649122807017,
+              "15": 0.8272727272727273,
+              "16": 0.7606837606837606,
+              "17": 0.6956521739130435,
+              "18": 0.5794117647058824,
+              "19": 0.5744047619047619,
+              "20": 0.5209580838323353
+            },
+            "pass@64": {
+              "11": 0.9728915662650602,
+              "12": 0.9780564263322884,
+              "13": 0.969187675070028,
+              "14": 0.9502923976608187,
+              "15": 0.896969696969697,
+              "16": 0.8034188034188035,
+              "17": 0.7795031055900621,
+              "18": 0.6823529411764706,
+              "19": 0.6190476190476191,
+              "20": 0.5658682634730539
+            },
+            "pass@128": {
+              "11": 0.9789156626506024,
+              "12": 0.9811912225705329,
+              "13": 0.9915966386554622,
+              "14": 0.9736842105263158,
+              "15": 0.9454545454545454,
+              "16": 0.8376068376068376,
+              "17": 0.8136645962732919,
+              "18": 0.7529411764705882,
+              "19": 0.6755952380952381,
+              "20": 0.6287425149700598
+            }
+          },
+          "total_pass_at_k": {
+            "pass@1": 0.5031222123104371,
+            "pass@2": 0.5450490633363069,
+            "pass@4": 0.6024382991376747,
+            "pass@8": 0.6645851917930419,
+            "pass@16": 0.7178114778471603,
+            "pass@32": 0.7719298245614035,
+            "pass@64": 0.8218852215283973,
+            "pass@128": 0.8581623550401427
+          }
+        }
+      },
+      "total": {
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.618,
+            "pass@2": 0.682,
+            "pass@4": 0.773,
+            "pass@8": 0.856,
+            "pass@16": 0.925,
+            "pass@32": 0.962,
+            "pass@64": 0.974,
+            "pass@128": 0.984
+          },
+          "12": {
+            "pass@1": 0.621,
+            "pass@2": 0.683,
+            "pass@4": 0.772,
+            "pass@8": 0.862,
+            "pass@16": 0.921,
+            "pass@32": 0.956,
+            "pass@64": 0.975,
+            "pass@128": 0.982
+          },
+          "13": {
+            "pass@1": 0.593,
+            "pass@2": 0.665,
+            "pass@4": 0.75,
+            "pass@8": 0.837,
+            "pass@16": 0.896,
+            "pass@32": 0.932,
+            "pass@64": 0.957,
+            "pass@128": 0.975
+          },
+          "14": {
+            "pass@1": 0.563,
+            "pass@2": 0.605,
+            "pass@4": 0.66,
+            "pass@8": 0.727,
+            "pass@16": 0.787,
+            "pass@32": 0.863,
+            "pass@64": 0.921,
+            "pass@128": 0.947
+          },
+          "15": {
+            "pass@1": 0.517,
+            "pass@2": 0.557,
+            "pass@4": 0.595,
+            "pass@8": 0.667,
+            "pass@16": 0.736,
+            "pass@32": 0.818,
+            "pass@64": 0.895,
+            "pass@128": 0.941
+          },
+          "16": {
+            "pass@1": 0.525,
+            "pass@2": 0.562,
+            "pass@4": 0.599,
+            "pass@8": 0.648,
+            "pass@16": 0.696,
+            "pass@32": 0.749,
+            "pass@64": 0.793,
+            "pass@128": 0.823
+          },
+          "17": {
+            "pass@1": 0.489,
+            "pass@2": 0.514,
+            "pass@4": 0.555,
+            "pass@8": 0.603,
+            "pass@16": 0.659,
+            "pass@32": 0.716,
+            "pass@64": 0.779,
+            "pass@128": 0.823
+          },
+          "18": {
+            "pass@1": 0.433,
+            "pass@2": 0.471,
+            "pass@4": 0.497,
+            "pass@8": 0.532,
+            "pass@16": 0.587,
+            "pass@32": 0.63,
+            "pass@64": 0.708,
+            "pass@128": 0.772
+          },
+          "19": {
+            "pass@1": 0.365,
+            "pass@2": 0.422,
+            "pass@4": 0.449,
+            "pass@8": 0.492,
+            "pass@16": 0.525,
+            "pass@32": 0.571,
+            "pass@64": 0.613,
+            "pass@128": 0.659
+          },
+          "20": {
+            "pass@1": 0.284,
+            "pass@2": 0.337,
+            "pass@4": 0.37,
+            "pass@8": 0.415,
+            "pass@16": 0.46,
+            "pass@32": 0.516,
+            "pass@64": 0.567,
+            "pass@128": 0.621
+          }
+        },
+        "per_op_pass_at_columns": {
+          "pass@1": {
+            "11": 0.618,
+            "12": 0.621,
+            "13": 0.593,
+            "14": 0.563,
+            "15": 0.517,
+            "16": 0.525,
+            "17": 0.489,
+            "18": 0.433,
+            "19": 0.365,
+            "20": 0.284
+          },
+          "pass@2": {
+            "11": 0.682,
+            "12": 0.683,
+            "13": 0.665,
+            "14": 0.605,
+            "15": 0.557,
+            "16": 0.562,
+            "17": 0.514,
+            "18": 0.471,
+            "19": 0.422,
+            "20": 0.337
+          },
+          "pass@4": {
+            "11": 0.773,
+            "12": 0.772,
+            "13": 0.75,
+            "14": 0.66,
+            "15": 0.595,
+            "16": 0.599,
+            "17": 0.555,
+            "18": 0.497,
+            "19": 0.449,
+            "20": 0.37
+          },
+          "pass@8": {
+            "11": 0.856,
+            "12": 0.862,
+            "13": 0.837,
+            "14": 0.727,
+            "15": 0.667,
+            "16": 0.648,
+            "17": 0.603,
+            "18": 0.532,
+            "19": 0.492,
+            "20": 0.415
+          },
+          "pass@16": {
+            "11": 0.925,
+            "12": 0.921,
+            "13": 0.896,
+            "14": 0.787,
+            "15": 0.736,
+            "16": 0.696,
+            "17": 0.659,
+            "18": 0.587,
+            "19": 0.525,
+            "20": 0.46
+          },
+          "pass@32": {
+            "11": 0.962,
+            "12": 0.956,
+            "13": 0.932,
+            "14": 0.863,
+            "15": 0.818,
+            "16": 0.749,
+            "17": 0.716,
+            "18": 0.63,
+            "19": 0.571,
+            "20": 0.516
+          },
+          "pass@64": {
+            "11": 0.974,
+            "12": 0.975,
+            "13": 0.957,
+            "14": 0.921,
+            "15": 0.895,
+            "16": 0.793,
+            "17": 0.779,
+            "18": 0.708,
+            "19": 0.613,
+            "20": 0.567
+          },
+          "pass@128": {
+            "11": 0.984,
+            "12": 0.982,
+            "13": 0.975,
+            "14": 0.947,
+            "15": 0.941,
+            "16": 0.823,
+            "17": 0.823,
+            "18": 0.772,
+            "19": 0.659,
+            "20": 0.621
+          }
+        },
+        "total_pass_at_k": {
+          "pass@1": 0.5008,
+          "pass@2": 0.5498,
+          "pass@4": 0.602,
+          "pass@8": 0.6639,
+          "pass@16": 0.7192,
+          "pass@32": 0.7713,
+          "pass@64": 0.8182,
+          "pass@128": 0.8527
+        }
+      },
+      "ks": [
+        1,
+        2,
+        4,
+        8,
+        16,
+        32,
+        64,
+        128
+      ],
+      "prompt_counts": {
+        "per_template": {
+          "crazy_zootopia": {
+            "per_op": {
+              "11": 353,
+              "12": 354,
+              "13": 319,
+              "14": 321,
+              "15": 346,
+              "16": 310,
+              "17": 346,
+              "18": 350,
+              "19": 333,
+              "20": 333
+            },
+            "total": 3365
+          },
+          "movie_festival_awards": {
+            "per_op": {
+              "11": 315,
+              "12": 327,
+              "13": 324,
+              "14": 337,
+              "15": 324,
+              "16": 339,
+              "17": 332,
+              "18": 310,
+              "19": 331,
+              "20": 333
+            },
+            "total": 3272
+          },
+          "teachers_in_school": {
+            "per_op": {
+              "11": 332,
+              "12": 319,
+              "13": 357,
+              "14": 342,
+              "15": 330,
+              "16": 351,
+              "17": 322,
+              "18": 340,
+              "19": 336,
+              "20": 334
+            },
+            "total": 3363
+          }
+        },
+        "total": {
+          "per_op": {
+            "11": 1000,
+            "12": 1000,
+            "13": 1000,
+            "14": 1000,
+            "15": 1000,
+            "16": 1000,
+            "17": 1000,
+            "18": 1000,
+            "19": 1000,
+            "20": 1000
+          },
+          "total": 10000
+        }
+      }
+    }
+  }
+}

Qwen3-100M-midtrain-diff_ar-block4/composition/summary.csv

@@ -0,0 +1,2 @@
+checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total
+checkpoint-38646,0.9930946180555555,0.870721875,0.9286879111842106,0.0,0.0,0.0,200.28550694444445,379.828709375,1152000,1280000,2432000

Qwen3-100M-midtrain-diff_ar-block4/composition/summary.json

@@ -0,0 +1,645 @@
+[
+  {
+    "checkpoint": "checkpoint-38646",
+    "id_acc": 0.9930946180555555,
+    "ood_acc": 0.870721875,
+    "total_acc": 0.9286879111842106,
+    "id_avg_loss": 0.0,
+    "ood_avg_loss": 0.0,
+    "total_avg_loss": 0.0,
+    "id_avg_resp_len": 200.28550694444445,
+    "ood_avg_resp_len": 379.828709375,
+    "count_id": 1152000,
+    "count_ood": 1280000,
+    "count_total": 2432000,
+    "template_metrics": {
+      "crazy_zootopia": {
+        "count": 809728,
+        "correct": 747714,
+        "answer_accuracy": 0.9234137883338602,
+        "avg_response_len": 281.5405074296554,
+        "resp_tokens_sum": 227971232,
+        "pass_at_k": {
+          "pass@1": 0.9234137883338602,
+          "pass@2": 0.9435549933283615,
+          "pass@4": 0.9572698152890534,
+          "pass@8": 0.9667895172243828,
+          "pass@16": 0.9736588935734707,
+          "pass@32": 0.9790904995462497,
+          "pass@64": 0.9834172709208217,
+          "pass@128": 0.9867214669617452
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.954484697640118,
+            "pass@2": 0.9629707860079438,
+            "pass@4": 0.9709464638269774,
+            "pass@8": 0.9782191819839355,
+            "pass@16": 0.984593484638989,
+            "pass@32": 0.9892985145351701,
+            "pass@64": 0.9930035863666121,
+            "pass@128": 0.9970501474926253
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9980902777777778,
+            "pass@2": 0.9997808867641542,
+            "pass@4": 0.9999950205629058,
+            "pass@8": 0.9999999974305525,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9998830464071856,
+            "pass@2": 0.9999977898533642,
+            "pass@4": 0.9999999997193464,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9991910133136095,
+            "pass@2": 0.9999253803056423,
+            "pass@4": 0.9999992966855474,
+            "pass@8": 0.9999999999707041,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9977964743589743,
+            "pass@2": 0.9990386192711489,
+            "pass@4": 0.9997245203123646,
+            "pass@8": 0.9999788364839716,
+            "pass@16": 0.9999999148409177,
+            "pass@32": 0.9999999999998113,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9962367957746479,
+            "pass@2": 0.9972319646223802,
+            "pass@4": 0.9975970072402921,
+            "pass@8": 0.9979708642982112,
+            "pass@16": 0.9985716335094632,
+            "pass@32": 0.9993491975474896,
+            "pass@64": 0.9999188200066541,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9682852337110481,
+            "pass@2": 0.977452822823493,
+            "pass@4": 0.9842284054082472,
+            "pass@8": 0.9891870968251745,
+            "pass@16": 0.9931120473781875,
+            "pass@32": 0.9965309204049758,
+            "pass@64": 0.9991380576749798,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9647775423728814,
+            "pass@2": 0.9793001245607014,
+            "pass@4": 0.9885038096403199,
+            "pass@8": 0.9930489687895803,
+            "pass@16": 0.9957842683111874,
+            "pass@32": 0.9978504659297166,
+            "pass@64": 0.9992600866784821,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9408797021943573,
+            "pass@2": 0.9649356379433759,
+            "pass@4": 0.9808571087704949,
+            "pass@8": 0.9881229406714634,
+            "pass@16": 0.99055864455524,
+            "pass@32": 0.9913680560096522,
+            "pass@64": 0.9921630087220765,
+            "pass@128": 0.9937304075235109
+          },
+          "14": {
+            "pass@1": 0.9360640576323987,
+            "pass@2": 0.9562122856722352,
+            "pass@4": 0.9707606814919159,
+            "pass@8": 0.9804801905602375,
+            "pass@16": 0.9866715910082771,
+            "pass@32": 0.9909012287981317,
+            "pass@64": 0.9945214531172304,
+            "pass@128": 0.9968847352024922
+          },
+          "15": {
+            "pass@1": 0.9660404624277457,
+            "pass@2": 0.9819844198033775,
+            "pass@4": 0.989337811097312,
+            "pass@8": 0.9927511350152156,
+            "pass@16": 0.9945051007958996,
+            "pass@32": 0.9954700452049894,
+            "pass@64": 0.996392965878996,
+            "pass@128": 0.9971098265895953
+          },
+          "16": {
+            "pass@1": 0.8782762096774194,
+            "pass@2": 0.9086745142240289,
+            "pass@4": 0.9307146733674418,
+            "pass@8": 0.9456327971297461,
+            "pass@16": 0.9552193362938507,
+            "pass@32": 0.9633343766463572,
+            "pass@64": 0.9715414297733572,
+            "pass@128": 0.9806451612903225
+          },
+          "17": {
+            "pass@1": 0.8934248554913294,
+            "pass@2": 0.9237297175822676,
+            "pass@4": 0.9422482222236664,
+            "pass@8": 0.9540871391786083,
+            "pass@16": 0.9620315216397886,
+            "pass@32": 0.9683743703109818,
+            "pass@64": 0.9725996832538545,
+            "pass@128": 0.9739884393063584
+          },
+          "18": {
+            "pass@1": 0.8005357142857142,
+            "pass@2": 0.8531492547806522,
+            "pass@4": 0.8860468795864797,
+            "pass@8": 0.9091291155842222,
+            "pass@16": 0.9275554499538204,
+            "pass@32": 0.9442031158786668,
+            "pass@64": 0.9575396478067062,
+            "pass@128": 0.9657142857142857
+          },
+          "19": {
+            "pass@1": 0.6647428678678678,
+            "pass@2": 0.7474041462722564,
+            "pass@4": 0.8052284390689899,
+            "pass@8": 0.8453281707689558,
+            "pass@16": 0.8732022792173314,
+            "pass@32": 0.8947298964394246,
+            "pass@64": 0.9117678285429259,
+            "pass@128": 0.924924924924925
+          },
+          "20": {
+            "pass@1": 0.5938203828828829,
+            "pass@2": 0.6811821664183864,
+            "pass@4": 0.7462476041846112,
+            "pass@8": 0.7985336624704658,
+            "pass@16": 0.8402692553045812,
+            "pass@32": 0.8729885399962004,
+            "pass@64": 0.8982468595231101,
+            "pass@128": 0.918918918918919
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 821120,
+        "correct": 765697,
+        "answer_accuracy": 0.9325031664068589,
+        "avg_response_len": 291.1401585639127,
+        "resp_tokens_sum": 239061007,
+        "pass_at_k": {
+          "pass@1": 0.9325031664068589,
+          "pass@2": 0.9518363817271153,
+          "pass@4": 0.9646200336464147,
+          "pass@8": 0.9733466730553039,
+          "pass@16": 0.9793728174194902,
+          "pass@32": 0.983754479786111,
+          "pass@64": 0.9872180069196503,
+          "pass@128": 0.9900233826968043
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9433713573619632,
+            "pass@2": 0.9535782540698516,
+            "pass@4": 0.9618546829460732,
+            "pass@8": 0.968359742151891,
+            "pass@16": 0.9741511694787096,
+            "pass@32": 0.9805862212217205,
+            "pass@64": 0.9881224160445984,
+            "pass@128": 0.9938650306748467
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9983901515151515,
+            "pass@2": 0.999460898353615,
+            "pass@4": 0.9999225821204167,
+            "pass@8": 0.9999984049427112,
+            "pass@16": 0.9999999996635676,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.999791974852071,
+            "pass@2": 0.9999868960536739,
+            "pass@4": 0.9999999650561431,
+            "pass@8": 0.9999999999999813,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9935914855072464,
+            "pass@2": 0.9972134257674314,
+            "pass@4": 0.9991986126734157,
+            "pass@8": 0.9998860811105759,
+            "pass@16": 0.9999969943734291,
+            "pass@32": 0.9999999992176973,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999328438395415,
+            "pass@2": 0.99999894242267,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9981484826589595,
+            "pass@2": 0.9991423353511447,
+            "pass@4": 0.9997656553768929,
+            "pass@8": 0.9999830971478822,
+            "pass@16": 0.9999999407613452,
+            "pass@32": 0.9999999999999076,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9978213028169014,
+            "pass@2": 0.998696212709327,
+            "pass@4": 0.9994122448602376,
+            "pass@8": 0.9998848700630322,
+            "pass@16": 0.9999964254002791,
+            "pass@32": 0.9999999987973975,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9976609281437125,
+            "pass@2": 0.9988149930454053,
+            "pass@4": 0.9995776498184733,
+            "pass@8": 0.9999453661202221,
+            "pass@16": 0.9999993082440582,
+            "pass@32": 0.9999999999733661,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9791274472891566,
+            "pass@2": 0.9844943257281095,
+            "pass@4": 0.9885222114554959,
+            "pass@8": 0.9915842526776513,
+            "pass@16": 0.9936573596958077,
+            "pass@32": 0.9946784629745986,
+            "pass@64": 0.9954818436151764,
+            "pass@128": 0.9969879518072289
+          },
+          "12": {
+            "pass@1": 0.9556720219435737,
+            "pass@2": 0.9739207939426853,
+            "pass@4": 0.9861632180381843,
+            "pass@8": 0.9932283805113136,
+            "pass@16": 0.9967175800031935,
+            "pass@32": 0.9986257066354457,
+            "pass@64": 0.9997926928260641,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9692752100840336,
+            "pass@2": 0.9822620977524867,
+            "pass@4": 0.9899195350056031,
+            "pass@8": 0.9950258375460176,
+            "pass@16": 0.9984359616586164,
+            "pass@32": 0.9998247453476256,
+            "pass@64": 0.9999989636921891,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9544956140350878,
+            "pass@2": 0.9714041706957683,
+            "pass@4": 0.9850686652107085,
+            "pass@8": 0.9941410138686707,
+            "pass@16": 0.9982549357238066,
+            "pass@32": 0.9997482243275714,
+            "pass@64": 0.9999976075223646,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.962405303030303,
+            "pass@2": 0.9801527827487472,
+            "pass@4": 0.9897256777561895,
+            "pass@8": 0.9949554819709671,
+            "pass@16": 0.9975806792718727,
+            "pass@32": 0.9989957428151091,
+            "pass@64": 0.999819329898887,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.9157540954415955,
+            "pass@2": 0.9383107460125177,
+            "pass@4": 0.9528681665860143,
+            "pass@8": 0.9618255529076115,
+            "pass@16": 0.9666002487818389,
+            "pass@32": 0.9691287175972142,
+            "pass@64": 0.9713401996282022,
+            "pass@128": 0.9743589743589743
+          },
+          "17": {
+            "pass@1": 0.8866459627329193,
+            "pass@2": 0.9140307869125057,
+            "pass@4": 0.9320510835796145,
+            "pass@8": 0.9470251241563546,
+            "pass@16": 0.958828933724438,
+            "pass@32": 0.9669829599693496,
+            "pass@64": 0.9726542282720683,
+            "pass@128": 0.9782608695652174
+          },
+          "18": {
+            "pass@1": 0.8592830882352941,
+            "pass@2": 0.9078276980083376,
+            "pass@4": 0.9323671411477975,
+            "pass@8": 0.9477004466579106,
+            "pass@16": 0.9584144161429146,
+            "pass@32": 0.9654229587125491,
+            "pass@64": 0.9701825020374101,
+            "pass@128": 0.9735294117647059
+          },
+          "19": {
+            "pass@1": 0.6903366815476191,
+            "pass@2": 0.7712544232752157,
+            "pass@4": 0.8257569664841,
+            "pass@8": 0.8643543757323078,
+            "pass@16": 0.8929841094671906,
+            "pass@32": 0.914312749232494,
+            "pass@64": 0.9313348502509535,
+            "pass@128": 0.9464285714285714
+          },
+          "20": {
+            "pass@1": 0.6061938622754491,
+            "pass@2": 0.7069592360554483,
+            "pass@4": 0.779909319718268,
+            "pass@8": 0.8312956919046384,
+            "pass@16": 0.8691291374054729,
+            "pass@32": 0.9006392983887458,
+            "pass@64": 0.926877074969234,
+            "pass@128": 0.9461077844311377
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 801152,
+        "correct": 745158,
+        "answer_accuracy": 0.930108144272248,
+        "avg_response_len": 311.89763365753316,
+        "resp_tokens_sum": 249877413,
+        "pass_at_k": {
+          "pass@1": 0.930108144272248,
+          "pass@2": 0.9506625813159659,
+          "pass@4": 0.9639841486302464,
+          "pass@8": 0.9728187372779841,
+          "pass@16": 0.9789563417603896,
+          "pass@32": 0.9835196674940303,
+          "pass@64": 0.9868039392795732,
+          "pass@128": 0.9892954146029718
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9640391791044776,
+            "pass@2": 0.9751006287460335,
+            "pass@4": 0.9840680005484389,
+            "pass@8": 0.989525217666148,
+            "pass@16": 0.9922932987785924,
+            "pass@32": 0.9943696315299635,
+            "pass@64": 0.9970265954859788,
+            "pass@128": 1.0
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.9998857821637427,
+            "pass@2": 0.9999978415527007,
+            "pass@4": 0.9999999997259115,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9920266544117647,
+            "pass@2": 0.9942518961324689,
+            "pass@4": 0.9955414982134585,
+            "pass@8": 0.9967116901009395,
+            "pass@16": 0.998155080969405,
+            "pass@32": 0.9994198334387632,
+            "pass@64": 0.9999589349376307,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9999260646687698,
+            "pass@2": 0.9999988356640751,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9976513053797469,
+            "pass@2": 0.9984395245689226,
+            "pass@4": 0.9992420280930073,
+            "pass@8": 0.9998284242652762,
+            "pass@16": 0.9999927222555876,
+            "pass@32": 0.9999999948561573,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9929147897897898,
+            "pass@2": 0.9970771706746117,
+            "pass@4": 0.9993902071362702,
+            "pass@8": 0.9999651367971397,
+            "pass@16": 0.9999998583663093,
+            "pass@32": 0.9999999999994196,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9923884646302251,
+            "pass@2": 0.9939457427146365,
+            "pass@4": 0.994643188391644,
+            "pass@8": 0.995451850487162,
+            "pass@16": 0.9964930745316808,
+            "pass@32": 0.9974718962737498,
+            "pass@64": 0.9983913004056533,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9703125,
+            "pass@2": 0.9826138920134985,
+            "pass@4": 0.9905110777819439,
+            "pass@8": 0.9952361673715535,
+            "pass@16": 0.9982486278795625,
+            "pass@32": 0.9997469296363813,
+            "pass@64": 0.9999980432330696,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9693950688073395,
+            "pass@2": 0.9864906149919334,
+            "pass@4": 0.9964843422668495,
+            "pass@8": 0.999587940904666,
+            "pass@16": 0.9999840639525752,
+            "pass@32": 0.9999999728189605,
+            "pass@64": 0.9999999999999997,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9416232638888888,
+            "pass@2": 0.9657007934772044,
+            "pass@4": 0.9807125428765847,
+            "pass@8": 0.9889479333237797,
+            "pass@16": 0.9935783281704044,
+            "pass@32": 0.9965795260843454,
+            "pass@64": 0.9983950138936485,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9647626112759644,
+            "pass@2": 0.9797271840463564,
+            "pass@4": 0.988019856104782,
+            "pass@8": 0.992716483465879,
+            "pass@16": 0.9955701659139958,
+            "pass@32": 0.9973444053304108,
+            "pass@64": 0.9984967309084293,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.9497010030864198,
+            "pass@2": 0.9660535597112858,
+            "pass@4": 0.9750617191369597,
+            "pass@8": 0.9813197913201593,
+            "pass@16": 0.9856122764931645,
+            "pass@32": 0.9887173282223192,
+            "pass@64": 0.991433089630488,
+            "pass@128": 0.9938271604938271
+          },
+          "16": {
+            "pass@1": 0.8941970870206489,
+            "pass@2": 0.9212493177014375,
+            "pass@4": 0.9393702139555561,
+            "pass@8": 0.9519523737165061,
+            "pass@16": 0.9612349386592156,
+            "pass@32": 0.9691753292432682,
+            "pass@64": 0.9747878703756769,
+            "pass@128": 0.976401179941003
+          },
+          "17": {
+            "pass@1": 0.8594455948795181,
+            "pass@2": 0.897720433900958,
+            "pass@4": 0.922174053073938,
+            "pass@8": 0.9382293341864976,
+            "pass@16": 0.9493378639982005,
+            "pass@32": 0.959124887388815,
+            "pass@64": 0.9681289020289342,
+            "pass@128": 0.9759036144578314
+          },
+          "18": {
+            "pass@1": 0.8485887096774194,
+            "pass@2": 0.903131746888494,
+            "pass@4": 0.9370043603420539,
+            "pass@8": 0.9578532946585311,
+            "pass@16": 0.9702169368171149,
+            "pass@32": 0.9777537348282815,
+            "pass@64": 0.9826360654372754,
+            "pass@128": 0.9870967741935484
+          },
+          "19": {
+            "pass@1": 0.7076331193353474,
+            "pass@2": 0.7887924625924787,
+            "pass@4": 0.8411277290300948,
+            "pass@8": 0.878659443062788,
+            "pass@16": 0.9053831900865986,
+            "pass@32": 0.9237568606242907,
+            "pass@64": 0.9352225250892545,
+            "pass@128": 0.9425981873111783
+          },
+          "20": {
+            "pass@1": 0.626196509009009,
+            "pass@2": 0.7125315522215124,
+            "pass@4": 0.7734252413268156,
+            "pass@8": 0.8190036582565382,
+            "pass@16": 0.8555370910100033,
+            "pass@32": 0.8847167374444287,
+            "pass@64": 0.9059201413232556,
+            "pass@128": 0.9219219219219219
+          }
+        }
+      }
+    }
+  }
+]

Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_id_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bc96c6c038c1ffd5920d3659d33c7102aa384e31647ed37d10c1b8001dd287d
+size 1789699126

Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_metrics.json

@@ -0,0 +1,1897 @@
+{
+  "checkpoint": "checkpoint-38645",
+  "id": {
+    "answer_accuracy": 0.9926675347222222,
+    "avg_response_len": 200.20069097222222,
+    "avg_loss": 0.0,
+    "count": 1152000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.9926675347222222,
+      "pass@2": 0.9947181075021866,
+      "pass@4": 0.9961613556430446,
+      "pass@8": 0.9973026695761072,
+      "pass@16": 0.9981802787310782,
+      "pass@32": 0.9987948199302888,
+      "pass@64": 0.9992436062012987,
+      "pass@128": 0.9995555555555555
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.9546875,
+        "pass@2": 0.9643489173228345,
+        "pass@4": 0.9724239796587925,
+        "pass@8": 0.9796451423626065,
+        "pass@16": 0.9858481869460654,
+        "pass@32": 0.9905653033141515,
+        "pass@64": 0.9942351004838736,
+        "pass@128": 0.997
+      },
+      "2": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "3": {
+        "pass@1": 0.99915625,
+        "pass@2": 0.9997679625984253,
+        "pass@4": 0.9999731251406074,
+        "pass@8": 0.9999995472606525,
+        "pass@16": 0.9999999999231317,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "4": {
+        "pass@1": 0.99971875,
+        "pass@2": 0.9999778543307086,
+        "pass@4": 0.9999998386764155,
+        "pass@8": 0.9999999999969994,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "5": {
+        "pass@1": 0.9941953125,
+        "pass@2": 0.9956927903543309,
+        "pass@4": 0.9968433910761155,
+        "pass@8": 0.9977670040034647,
+        "pass@16": 0.9983917733914977,
+        "pass@32": 0.9987650573014524,
+        "pass@64": 0.9989711801510246,
+        "pass@128": 0.999
+      },
+      "6": {
+        "pass@1": 0.99975,
+        "pass@2": 0.9999803149606299,
+        "pass@4": 0.9999997736220472,
+        "pass@8": 0.9999999999829965,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "7": {
+        "pass@1": 0.9958359375,
+        "pass@2": 0.9972054625984251,
+        "pass@4": 0.9982031517622797,
+        "pass@8": 0.9990104842344203,
+        "pass@16": 0.9995193774344288,
+        "pass@32": 0.9998284427302075,
+        "pass@64": 0.9999861762904964,
+        "pass@128": 1.0
+      },
+      "8": {
+        "pass@1": 0.9950703125,
+        "pass@2": 0.997607160433071,
+        "pass@4": 0.9989324190101235,
+        "pass@8": 0.9996309201327338,
+        "pass@16": 0.9999226231371163,
+        "pass@32": 0.9999963916027095,
+        "pass@64": 0.9999999990502599,
+        "pass@128": 1.0
+      },
+      "9": {
+        "pass@1": 0.99559375,
+        "pass@2": 0.9978825049212597,
+        "pass@4": 0.9990765218410199,
+        "pass@8": 0.9996709282110962,
+        "pass@16": 0.9999405477474613,
+        "pass@32": 0.9999981844240781,
+        "pass@64": 0.9999999998360332,
+        "pass@128": 1.0
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.9546875,
+      "2": 1.0,
+      "3": 0.99915625,
+      "4": 0.99971875,
+      "5": 0.9941953125,
+      "6": 0.99975,
+      "7": 0.9958359375,
+      "8": 0.9950703125,
+      "9": 0.99559375
+    },
+    "per_op_avg_response_len": {
+      "10": 281.6347734375,
+      "2": 115.6326015625,
+      "3": 144.829234375,
+      "4": 159.923921875,
+      "5": 173.2866484375,
+      "6": 198.9441953125,
+      "7": 224.1663515625,
+      "8": 243.7276171875,
+      "9": 259.660875
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 281.6347734375,
+      "2": 115.6326015625,
+      "3": 144.829234375,
+      "4": 159.923921875,
+      "5": 173.2866484375,
+      "6": 198.9441953125,
+      "7": 224.1663515625,
+      "8": 243.7276171875,
+      "9": 259.660875
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 379008,
+        "correct": 376612,
+        "answer_accuracy": 0.9936782337048294,
+        "avg_response_len": 190.71223826409997,
+        "resp_tokens_sum": 72281464,
+        "pass_at_k": {
+          "pass@1": 0.9936782337048294,
+          "pass@2": 0.9956697567059437,
+          "pass@4": 0.996967571777692,
+          "pass@8": 0.9979938409459532,
+          "pass@16": 0.9988097707718845,
+          "pass@32": 0.9992490341086927,
+          "pass@64": 0.9994945000973269,
+          "pass@128": 0.999662276258021
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9615366887905604,
+            "pass@2": 0.970145808654449,
+            "pass@4": 0.9771259859110534,
+            "pass@8": 0.9837142589656627,
+            "pass@16": 0.9897806295670789,
+            "pass@32": 0.9934434017257641,
+            "pass@64": 0.9955847044260153,
+            "pass@128": 0.9970501474926253
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9997610856269113,
+            "pass@2": 0.9999864552481399,
+            "pass@4": 0.9999999638806617,
+            "pass@8": 0.9999999999999809,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.99931640625,
+            "pass@2": 0.9999350239911416,
+            "pass@4": 0.9999994973284589,
+            "pass@8": 0.999999999990623,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9954861111111111,
+            "pass@2": 0.9972468285214349,
+            "pass@4": 0.9985486585010207,
+            "pass@8": 0.9994671341886722,
+            "pass@16": 0.9999236250321564,
+            "pass@32": 0.9999990493973,
+            "pass@64": 0.9999999999949286,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9995555763473054,
+            "pass@2": 0.9999499033429206,
+            "pass@4": 0.9999993320445721,
+            "pass@8": 0.9999999999490911,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9964404585798816,
+            "pass@2": 0.9978735207100592,
+            "pass@4": 0.9986706386183974,
+            "pass@8": 0.9994230191252826,
+            "pass@16": 0.9999007629980622,
+            "pass@32": 0.9999981593967747,
+            "pass@64": 0.99999999996853,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.997245592948718,
+            "pass@2": 0.9984009848828992,
+            "pass@4": 0.9992322474113812,
+            "pass@8": 0.9998262245762686,
+            "pass@16": 0.9999926289511722,
+            "pass@32": 0.9999999947902106,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9950704225352113,
+            "pass@2": 0.9983818759010756,
+            "pass@4": 0.999779442534472,
+            "pass@8": 0.9999935759719568,
+            "pass@16": 0.9999999938621692,
+            "pass@32": 0.9999999999999997,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 390656,
+        "correct": 387758,
+        "answer_accuracy": 0.9925817087155964,
+        "avg_response_len": 198.13595081094365,
+        "resp_tokens_sum": 77402998,
+        "pass_at_k": {
+          "pass@1": 0.9925817087155964,
+          "pass@2": 0.9942163103321948,
+          "pass@4": 0.9953949149315869,
+          "pass@8": 0.9964613581820646,
+          "pass@16": 0.9974663333818123,
+          "pass@32": 0.9984046891944619,
+          "pass@64": 0.999174331646805,
+          "pass@128": 0.9996723460026212
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9446654524539877,
+            "pass@2": 0.9550259498333415,
+            "pass@4": 0.9634138444542589,
+            "pass@8": 0.970866877957459,
+            "pass@16": 0.9782378206275941,
+            "pass@32": 0.9857900637806933,
+            "pass@64": 0.9923585282128631,
+            "pass@128": 0.9969325153374233
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9988873106060606,
+            "pass@2": 0.9997856269386781,
+            "pass@4": 0.9999895822681255,
+            "pass@8": 0.9999999777033803,
+            "pass@16": 0.9999999999999803,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.9999537721893491,
+            "pass@2": 0.999999636001491,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9965579710144927,
+            "pass@2": 0.9972633515919205,
+            "pass@4": 0.9975327839454851,
+            "pass@8": 0.9979120550709301,
+            "pass@16": 0.9985302315822023,
+            "pass@32": 0.9993303337082865,
+            "pass@64": 0.9999164669633687,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9997313753581661,
+            "pass@2": 0.9999915393813595,
+            "pass@4": 0.9999999905993125,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9975162572254336,
+            "pass@2": 0.9985822931364069,
+            "pass@4": 0.9993384239181086,
+            "pass@8": 0.9998572295938166,
+            "pass@16": 0.9999945349140869,
+            "pass@32": 0.9999999969725001,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9966769366197183,
+            "pass@2": 0.9978297937229678,
+            "pass@4": 0.9984739589769589,
+            "pass@8": 0.9991927685739189,
+            "pass@16": 0.9997893254704586,
+            "pass@32": 0.999989840105361,
+            "pass@64": 0.9999999973246758,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9973100673652695,
+            "pass@2": 0.9978701553585743,
+            "pass@4": 0.9984959510113631,
+            "pass@8": 0.9992628054872507,
+            "pass@16": 0.9998367378399854,
+            "pass@32": 0.9999945907477397,
+            "pass@64": 0.9999999995090815,
+            "pass@128": 1.0
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 382336,
+        "correct": 379183,
+        "answer_accuracy": 0.9917533269166388,
+        "avg_response_len": 211.7162234265149,
+        "resp_tokens_sum": 80946734,
+        "pass_at_k": {
+          "pass@1": 0.9917533269166388,
+          "pass@2": 0.9942874585803578,
+          "pass@4": 0.996145276324892,
+          "pass@8": 0.997477133569582,
+          "pass@16": 0.9982857475202078,
+          "pass@32": 0.9987431797640651,
+          "pass@64": 0.9990656782180966,
+          "pass@128": 0.9993304318714429
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9575093283582089,
+            "pass@2": 0.9675553090844993,
+            "pass@4": 0.9764339019189767,
+            "pass@8": 0.9840698697287039,
+            "pass@16": 0.9892746985021801,
+            "pass@32": 0.9922997890645133,
+            "pass@64": 0.9946955387643621,
+            "pass@128": 0.9970149253731343
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9988383746355685,
+            "pass@2": 0.9995426671793577,
+            "pass@4": 0.9999317049654508,
+            "pass@8": 0.999998701511788,
+            "pass@16": 0.9999999997759129,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.9998629385964912,
+            "pass@2": 0.9999964025878345,
+            "pass@4": 0.9999999986295574,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9906020220588235,
+            "pass@2": 0.9926593619731356,
+            "pass@4": 0.9945639799620636,
+            "pass@8": 0.9960446992193006,
+            "pass@16": 0.9968320635603195,
+            "pass@32": 0.997048210623365,
+            "pass@64": 0.9970588207360587,
+            "pass@128": 0.9970588235294118
+          },
+          "6": {
+            "pass@1": 0.9999753548895899,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9933494857594937,
+            "pass@2": 0.9949833518140139,
+            "pass@4": 0.9964600671949234,
+            "pass@8": 0.9976420959829561,
+            "pass@16": 0.9985911723443348,
+            "pass@32": 0.9994590693089006,
+            "pass@64": 0.9999562541175095,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9913194444444444,
+            "pass@2": 0.9966260551890473,
+            "pass@4": 0.9991402473902474,
+            "pass@8": 0.9999150306942846,
+            "pass@16": 0.9999991362232366,
+            "pass@32": 0.9999999999722541,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9943478697749196,
+            "pass@2": 0.9973257462592097,
+            "pass@4": 0.9988976594967428,
+            "pass@8": 0.9997409379688414,
+            "pass@16": 0.9999841784174797,
+            "pass@32": 0.999999971428724,
+            "pass@64": 0.9999999999999997,
+            "pass@128": 1.0
+          }
+        }
+      }
+    }
+  },
+  "ood": {
+    "answer_accuracy": 0.844671875,
+    "avg_response_len": 377.0967484375,
+    "avg_loss": 0.0,
+    "count": 1280000,
+    "validation_metrics": {},
+    "pass_at_k": {
+      "pass@1": 0.844671875,
+      "pass@2": 0.8858922244094257,
+      "pass@4": 0.9141995808398893,
+      "pass@8": 0.9341837080333515,
+      "pass@16": 0.9486755484175874,
+      "pass@32": 0.9595485287445626,
+      "pass@64": 0.9677227239017148,
+      "pass@128": 0.9738
+    },
+    "per_op_pass_at_k": {
+      "11": {
+        "pass@1": 0.9726171875,
+        "pass@2": 0.9828142224409449,
+        "pass@4": 0.9897272834645668,
+        "pass@8": 0.994115282605061,
+        "pass@16": 0.9967868462201568,
+        "pass@32": 0.9981416746464655,
+        "pass@64": 0.9987367902594374,
+        "pass@128": 0.999
+      },
+      "12": {
+        "pass@1": 0.9644375,
+        "pass@2": 0.9804958169291337,
+        "pass@4": 0.9905623012748407,
+        "pass@8": 0.9955742374047644,
+        "pass@16": 0.9976946487441701,
+        "pass@32": 0.998880341662968,
+        "pass@64": 0.9996771390867283,
+        "pass@128": 1.0
+      },
+      "13": {
+        "pass@1": 0.94846875,
+        "pass@2": 0.9692981053149601,
+        "pass@4": 0.9827727335958009,
+        "pass@8": 0.9904335216979527,
+        "pass@16": 0.9947883099927657,
+        "pass@32": 0.9972493649171777,
+        "pass@64": 0.9987378705005298,
+        "pass@128": 1.0
+      },
+      "14": {
+        "pass@1": 0.9449453125,
+        "pass@2": 0.964385088582677,
+        "pass@4": 0.9777694221034873,
+        "pass@8": 0.9865351007090998,
+        "pass@16": 0.9922419706883057,
+        "pass@32": 0.9962460277292379,
+        "pass@64": 0.9986672351003253,
+        "pass@128": 1.0
+      },
+      "15": {
+        "pass@1": 0.952953125,
+        "pass@2": 0.9735356791338584,
+        "pass@4": 0.9843826020809897,
+        "pass@8": 0.9903981612161823,
+        "pass@16": 0.9936567666222805,
+        "pass@32": 0.9955643444237633,
+        "pass@64": 0.9966508344175822,
+        "pass@128": 0.997
+      },
+      "16": {
+        "pass@1": 0.8934609375,
+        "pass@2": 0.9216697834645677,
+        "pass@4": 0.9393977767154105,
+        "pass@8": 0.9517931687352534,
+        "pass@16": 0.96139777462629,
+        "pass@32": 0.9693210349487325,
+        "pass@64": 0.9751328706018372,
+        "pass@128": 0.979
+      },
+      "17": {
+        "pass@1": 0.8661875,
+        "pass@2": 0.9038288631889774,
+        "pass@4": 0.9277408284589423,
+        "pass@8": 0.9448044024114316,
+        "pass@16": 0.9568565852772714,
+        "pass@32": 0.9646994651551266,
+        "pass@64": 0.9694077197479471,
+        "pass@128": 0.972
+      },
+      "18": {
+        "pass@1": 0.788859375,
+        "pass@2": 0.8561178641732294,
+        "pass@4": 0.8988476714473197,
+        "pass@8": 0.9267583655390567,
+        "pass@16": 0.9448361294806787,
+        "pass@32": 0.9568568515114735,
+        "pass@64": 0.9657552890983766,
+        "pass@128": 0.973
+      },
+      "19": {
+        "pass@1": 0.625640625,
+        "pass@2": 0.7169927411417328,
+        "pass@4": 0.7811060572740908,
+        "pass@8": 0.8285917496219818,
+        "pass@16": 0.8654698909087529,
+        "pass@32": 0.8941006950628526,
+        "pass@64": 0.9155477279248722,
+        "pass@128": 0.932
+      },
+      "20": {
+        "pass@1": 0.4891484375,
+        "pass@2": 0.5897840797244098,
+        "pass@4": 0.6696891319835014,
+        "pass@8": 0.7328330903927104,
+        "pass@16": 0.7830265616151493,
+        "pass@32": 0.824425487387794,
+        "pass@64": 0.8589137622794993,
+        "pass@128": 0.886
+      }
+    },
+    "per_op_accuracy": {
+      "11": 0.9726171875,
+      "12": 0.9644375,
+      "13": 0.94846875,
+      "14": 0.9449453125,
+      "15": 0.952953125,
+      "16": 0.8934609375,
+      "17": 0.8661875,
+      "18": 0.788859375,
+      "19": 0.625640625,
+      "20": 0.4891484375
+    },
+    "per_op_avg_response_len": {
+      "11": 300.8103125,
+      "12": 313.388375,
+      "13": 334.3829921875,
+      "14": 361.2994765625,
+      "15": 381.435625,
+      "16": 390.2542265625,
+      "17": 410.935984375,
+      "18": 425.279296875,
+      "19": 422.265296875,
+      "20": 430.9158984375
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "11": 300.8103125,
+      "12": 313.388375,
+      "13": 334.3829921875,
+      "14": 361.2994765625,
+      "15": 381.435625,
+      "16": 390.2542265625,
+      "17": 410.935984375,
+      "18": 425.279296875,
+      "19": 422.265296875,
+      "20": 430.9158984375
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 430720,
+        "correct": 361406,
+        "answer_accuracy": 0.8390741084695393,
+        "avg_response_len": 358.89673569836555,
+        "resp_tokens_sum": 154584002,
+        "pass_at_k": {
+          "pass@1": 0.8390741084695393,
+          "pass@2": 0.8810461662435247,
+          "pass@4": 0.910567308794798,
+          "pass@8": 0.9312514898839028,
+          "pass@16": 0.9461738981519632,
+          "pass@32": 0.9574585135295627,
+          "pass@64": 0.9658631665868088,
+          "pass@128": 0.9723625557206538
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9698123229461756,
+            "pass@2": 0.981584311079387,
+            "pass@4": 0.9891040887948495,
+            "pass@8": 0.9937797636155625,
+            "pass@16": 0.9970789483598115,
+            "pass@32": 0.999223562425918,
+            "pass@64": 0.9999570534856826,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9659030720338984,
+            "pass@2": 0.9809874632990792,
+            "pass@4": 0.9906215364498931,
+            "pass@8": 0.9956137072056146,
+            "pass@16": 0.9978803736651605,
+            "pass@32": 0.9993246177441057,
+            "pass@64": 0.9999568688255516,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9498922413793104,
+            "pass@2": 0.9708758608347934,
+            "pass@4": 0.9840897281060871,
+            "pass@8": 0.9909108589255634,
+            "pass@16": 0.9943999521503089,
+            "pass@32": 0.9963005415376021,
+            "pass@64": 0.9980499904733999,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9233839563862928,
+            "pass@2": 0.944520460789364,
+            "pass@4": 0.9611923819102052,
+            "pass@8": 0.9751079976868544,
+            "pass@16": 0.9861293435926446,
+            "pass@32": 0.9935272719026856,
+            "pass@64": 0.9975030863628863,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.9595375722543352,
+            "pass@2": 0.9784015719584908,
+            "pass@4": 0.987961396353635,
+            "pass@8": 0.9923778119086278,
+            "pass@16": 0.9938310119401556,
+            "pass@32": 0.9941908530213401,
+            "pass@64": 0.9942196182961773,
+            "pass@128": 0.9942196531791907
+          },
+          "16": {
+            "pass@1": 0.8793850806451613,
+            "pass@2": 0.9133687611125221,
+            "pass@4": 0.9350571322737883,
+            "pass@8": 0.9489252481008612,
+            "pass@16": 0.9594660322806725,
+            "pass@32": 0.9677192403291304,
+            "pass@64": 0.9722136540329196,
+            "pass@128": 0.9741935483870968
+          },
+          "17": {
+            "pass@1": 0.8896992413294798,
+            "pass@2": 0.9255197191752769,
+            "pass@4": 0.9464512051600483,
+            "pass@8": 0.9595738090472379,
+            "pass@16": 0.9673121876199731,
+            "pass@32": 0.9725731938539892,
+            "pass@64": 0.9768482847119901,
+            "pass@128": 0.9797687861271677
+          },
+          "18": {
+            "pass@1": 0.7444419642857143,
+            "pass@2": 0.8136916479190104,
+            "pass@4": 0.8625618206652741,
+            "pass@8": 0.8958082898451949,
+            "pass@16": 0.9174575405950798,
+            "pass@32": 0.9337781303810576,
+            "pass@64": 0.947397000627292,
+            "pass@128": 0.9571428571428572
+          },
+          "19": {
+            "pass@1": 0.6253753753753754,
+            "pass@2": 0.7184758577475104,
+            "pass@4": 0.7833039738275956,
+            "pass@8": 0.8296772377534075,
+            "pass@16": 0.8660549147719705,
+            "pass@32": 0.8940814351693145,
+            "pass@64": 0.9133406551952218,
+            "pass@128": 0.9279279279279279
+          },
+          "20": {
+            "pass@1": 0.4760932807807808,
+            "pass@2": 0.5768932071835613,
+            "pass@4": 0.6601997517765237,
+            "pass@8": 0.7265525943478015,
+            "pass@16": 0.7789762742387952,
+            "pass@32": 0.8215439955324653,
+            "pass@64": 0.8572865791166149,
+            "pass@128": 0.8888888888888888
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 430464,
+        "correct": 365670,
+        "answer_accuracy": 0.8494787020517395,
+        "avg_response_len": 372.46568586455544,
+        "resp_tokens_sum": 160333069,
+        "pass_at_k": {
+          "pass@1": 0.8494787020517395,
+          "pass@2": 0.8895784091467879,
+          "pass@4": 0.9172445349438386,
+          "pass@8": 0.9367987196255487,
+          "pass@16": 0.950993996091102,
+          "pass@32": 0.961825739478423,
+          "pass@64": 0.9700731776529568,
+          "pass@128": 0.976211715730003
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9770566641566265,
+            "pass@2": 0.9841648829570252,
+            "pass@4": 0.989164249235412,
+            "pass@8": 0.9922976666641117,
+            "pass@16": 0.9941890352635745,
+            "pass@32": 0.9952655846372421,
+            "pass@64": 0.9962408325530068,
+            "pass@128": 0.9969879518072289
+          },
+          "12": {
+            "pass@1": 0.9620640673981191,
+            "pass@2": 0.9773606620097254,
+            "pass@4": 0.9866140545127782,
+            "pass@8": 0.9920244775588233,
+            "pass@16": 0.9951791105256566,
+            "pass@32": 0.9972397387226369,
+            "pass@64": 0.9990357602586,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9582020308123249,
+            "pass@2": 0.9730195720020293,
+            "pass@4": 0.9829933805543213,
+            "pass@8": 0.9895737184859488,
+            "pass@16": 0.9944316023575817,
+            "pass@32": 0.9979028151511814,
+            "pass@64": 0.9996096734768417,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9545869883040936,
+            "pass@2": 0.9759829569001243,
+            "pass@4": 0.9895990122945156,
+            "pass@8": 0.9958761100329401,
+            "pass@16": 0.9982192698442767,
+            "pass@32": 0.9995253069447078,
+            "pass@64": 0.9999798903618302,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.964654356060606,
+            "pass@2": 0.9848753280839894,
+            "pass@4": 0.994307931394939,
+            "pass@8": 0.9984355730943542,
+            "pass@16": 0.9997865737709953,
+            "pass@32": 0.999996225078149,
+            "pass@64": 0.9999999999355342,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.9025106837606838,
+            "pass@2": 0.9249840865244409,
+            "pass@4": 0.9394740470475377,
+            "pass@8": 0.951603221594124,
+            "pass@16": 0.9618929894371758,
+            "pass@32": 0.9704062836352245,
+            "pass@64": 0.9772483460299233,
+            "pass@128": 0.9829059829059829
+          },
+          "17": {
+            "pass@1": 0.8692498059006211,
+            "pass@2": 0.9090858377757125,
+            "pass@4": 0.9353135909447651,
+            "pass@8": 0.9535516185363221,
+            "pass@16": 0.965466357509199,
+            "pass@32": 0.9717789256494813,
+            "pass@64": 0.9742932379404277,
+            "pass@128": 0.9751552795031055
+          },
+          "18": {
+            "pass@1": 0.8068244485294118,
+            "pass@2": 0.8750351001621121,
+            "pass@4": 0.9166487397714989,
+            "pass@8": 0.9411302798270424,
+            "pass@16": 0.9559175964520414,
+            "pass@32": 0.9659157855028336,
+            "pass@64": 0.9733500817730232,
+            "pass@128": 0.9794117647058823
+          },
+          "19": {
+            "pass@1": 0.6194428943452381,
+            "pass@2": 0.7078185051087363,
+            "pass@4": 0.7742094414537468,
+            "pass@8": 0.8272903079106774,
+            "pass@16": 0.8697325655416732,
+            "pass@32": 0.9032315125581434,
+            "pass@64": 0.9280735994123079,
+            "pass@128": 0.9464285714285714
+          },
+          "20": {
+            "pass@1": 0.47754491017964074,
+            "pass@2": 0.5815400449101793,
+            "pass@4": 0.6627141611041132,
+            "pass@8": 0.7251319869091699,
+            "pass@16": 0.7741932335037112,
+            "pass@32": 0.8160539579293532,
+            "pass@64": 0.8519449321090409,
+            "pass@128": 0.8802395209580839
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 418816,
+        "correct": 354104,
+        "answer_accuracy": 0.8454882334963325,
+        "avg_response_len": 400.5739202895782,
+        "resp_tokens_sum": 167766767,
+        "pass_at_k": {
+          "pass@1": 0.8454882334963325,
+          "pass@2": 0.8870873180698133,
+          "pass@4": 0.914805453327734,
+          "pass@8": 0.93451152896499,
+          "pass@16": 0.9488653753178875,
+          "pass@32": 0.9593574045087645,
+          "pass@64": 0.967219311432032,
+          "pass@128": 0.972799511002445
+        },
+        "per_op_pass_at_k": {
+          "11": {
+            "pass@1": 0.9710813492063493,
+            "pass@2": 0.9827689507561553,
+            "pass@4": 0.9910190773772323,
+            "pass@8": 0.9964069864008952,
+            "pass@16": 0.9991975166401181,
+            "pass@32": 0.9999605809858803,
+            "pass@64": 0.9999999808615654,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9651662844036697,
+            "pass@2": 0.9830220297864142,
+            "pass@4": 0.9943498286911382,
+            "pass@8": 0.9989944241978972,
+            "pass@16": 0.9999475847370622,
+            "pass@32": 0.9999998481010195,
+            "pass@64": 0.9999999999999678,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9363425925925926,
+            "pass@2": 0.9636441929133857,
+            "pass@4": 0.9812329428150182,
+            "pass@8": 0.9909109265469578,
+            "pass@16": 0.995563713627039,
+            "pass@32": 0.997463540610219,
+            "pass@64": 0.9984545373712423,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9556982566765578,
+            "pass@2": 0.971536645131896,
+            "pass@4": 0.9815543184142042,
+            "pass@8": 0.9879400706835424,
+            "pass@16": 0.9919984009149101,
+            "pass@32": 0.9955077699507003,
+            "pass@64": 0.9984439818222332,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.9340036651234568,
+            "pass@2": 0.9567897437299503,
+            "pass@4": 0.9704516715503154,
+            "pass@8": 0.9800978369588287,
+            "pass@16": 0.9872273677362894,
+            "pass@32": 0.9925171450697239,
+            "pass@64": 0.9958359460135142,
+            "pass@128": 0.9969135802469136
+          },
+          "16": {
+            "pass@1": 0.8969625737463127,
+            "pass@2": 0.9258290653380717,
+            "pass@4": 0.9432881274243372,
+            "pass@8": 0.9546124219600274,
+            "pass@16": 0.9626515201381516,
+            "pass@32": 0.9696621383207606,
+            "pass@64": 0.9756120011655725,
+            "pass@128": 0.9793510324483776
+          },
+          "17": {
+            "pass@1": 0.8387142319277109,
+            "pass@2": 0.8761247005739499,
+            "pass@4": 0.9008967927390097,
+            "pass@8": 0.9209284437120231,
+            "pass@16": 0.937609642297589,
+            "pass@32": 0.9496274880196173,
+            "pass@64": 0.9569150319903035,
+            "pass@128": 0.9608433734939759
+          },
+          "18": {
+            "pass@1": 0.819304435483871,
+            "pass@2": 0.8832704946659896,
+            "pass@4": 0.9202918151360103,
+            "pass@8": 0.9459392546840153,
+            "pass@16": 0.9635935725119584,
+            "pass@32": 0.9729778671198072,
+            "pass@64": 0.9781526163741832,
+            "pass@128": 0.9838709677419355
+          },
+          "19": {
+            "pass@1": 0.6321988293051359,
+            "pass@2": 0.7248134827651832,
+            "pass@4": 0.7858956545650826,
+            "pass@8": 0.828820803601541,
+            "pass@16": 0.8605542666697465,
+            "pass@32": 0.8848513260783547,
+            "pass@64": 0.905053052393738,
+            "pass@128": 0.9214501510574018
+          },
+          "20": {
+            "pass@1": 0.5138419669669669,
+            "pass@2": 0.6109437439407909,
+            "pass@4": 0.6861744289283656,
+            "pass@8": 0.7468378163580444,
+            "pass@16": 0.7959367036137914,
+            "pass@32": 0.8357036484296656,
+            "pass@64": 0.8675307030307697,
+            "pass@128": 0.8888888888888888
+          }
+        }
+      }
+    }
+  },
+  "total": {
+    "answer_accuracy": 0.9147750822368421,
+    "avg_loss": 0.0,
+    "count": 2432000,
+    "validation_metrics": {
+      "id": {},
+      "ood": {}
+    },
+    "pass_at_k": {
+      "pass@1": 0.9147750822368421,
+      "pass@2": 0.9374413269270534,
+      "pass@4": 0.9530235794308524,
+      "pass@8": 0.9640821635009731,
+      "pass@16": 0.9721251575134345,
+      "pass@32": 0.9781388772009632,
+      "pass@64": 0.9826536681488868,
+      "pass@128": 0.986
+    },
+    "per_op_pass_at_k": {
+      "10": {
+        "pass@1": 0.9546875,
+        "pass@2": 0.9643489173228345,
+        "pass@4": 0.9724239796587925,
+        "pass@8": 0.9796451423626065,
+        "pass@16": 0.9858481869460654,
+        "pass@32": 0.9905653033141515,
+        "pass@64": 0.9942351004838736,
+        "pass@128": 0.997
+      },
+      "2": {
+        "pass@1": 1.0,
+        "pass@2": 1.0,
+        "pass@4": 1.0,
+        "pass@8": 1.0,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "3": {
+        "pass@1": 0.99915625,
+        "pass@2": 0.9997679625984253,
+        "pass@4": 0.9999731251406074,
+        "pass@8": 0.9999995472606525,
+        "pass@16": 0.9999999999231317,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "4": {
+        "pass@1": 0.99971875,
+        "pass@2": 0.9999778543307086,
+        "pass@4": 0.9999998386764155,
+        "pass@8": 0.9999999999969994,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "5": {
+        "pass@1": 0.9941953125,
+        "pass@2": 0.9956927903543309,
+        "pass@4": 0.9968433910761155,
+        "pass@8": 0.9977670040034647,
+        "pass@16": 0.9983917733914977,
+        "pass@32": 0.9987650573014524,
+        "pass@64": 0.9989711801510246,
+        "pass@128": 0.999
+      },
+      "6": {
+        "pass@1": 0.99975,
+        "pass@2": 0.9999803149606299,
+        "pass@4": 0.9999997736220472,
+        "pass@8": 0.9999999999829965,
+        "pass@16": 1.0,
+        "pass@32": 1.0,
+        "pass@64": 1.0,
+        "pass@128": 1.0
+      },
+      "7": {
+        "pass@1": 0.9958359375,
+        "pass@2": 0.9972054625984251,
+        "pass@4": 0.9982031517622797,
+        "pass@8": 0.9990104842344203,
+        "pass@16": 0.9995193774344288,
+        "pass@32": 0.9998284427302075,
+        "pass@64": 0.9999861762904964,
+        "pass@128": 1.0
+      },
+      "8": {
+        "pass@1": 0.9950703125,
+        "pass@2": 0.997607160433071,
+        "pass@4": 0.9989324190101235,
+        "pass@8": 0.9996309201327338,
+        "pass@16": 0.9999226231371163,
+        "pass@32": 0.9999963916027095,
+        "pass@64": 0.9999999990502599,
+        "pass@128": 1.0
+      },
+      "9": {
+        "pass@1": 0.99559375,
+        "pass@2": 0.9978825049212597,
+        "pass@4": 0.9990765218410199,
+        "pass@8": 0.9996709282110962,
+        "pass@16": 0.9999405477474613,
+        "pass@32": 0.9999981844240781,
+        "pass@64": 0.9999999998360332,
+        "pass@128": 1.0
+      },
+      "11": {
+        "pass@1": 0.9726171875,
+        "pass@2": 0.9828142224409449,
+        "pass@4": 0.9897272834645668,
+        "pass@8": 0.994115282605061,
+        "pass@16": 0.9967868462201568,
+        "pass@32": 0.9981416746464655,
+        "pass@64": 0.9987367902594374,
+        "pass@128": 0.999
+      },
+      "12": {
+        "pass@1": 0.9644375,
+        "pass@2": 0.9804958169291337,
+        "pass@4": 0.9905623012748407,
+        "pass@8": 0.9955742374047644,
+        "pass@16": 0.9976946487441701,
+        "pass@32": 0.998880341662968,
+        "pass@64": 0.9996771390867283,
+        "pass@128": 1.0
+      },
+      "13": {
+        "pass@1": 0.94846875,
+        "pass@2": 0.9692981053149601,
+        "pass@4": 0.9827727335958009,
+        "pass@8": 0.9904335216979527,
+        "pass@16": 0.9947883099927657,
+        "pass@32": 0.9972493649171777,
+        "pass@64": 0.9987378705005298,
+        "pass@128": 1.0
+      },
+      "14": {
+        "pass@1": 0.9449453125,
+        "pass@2": 0.964385088582677,
+        "pass@4": 0.9777694221034873,
+        "pass@8": 0.9865351007090998,
+        "pass@16": 0.9922419706883057,
+        "pass@32": 0.9962460277292379,
+        "pass@64": 0.9986672351003253,
+        "pass@128": 1.0
+      },
+      "15": {
+        "pass@1": 0.952953125,
+        "pass@2": 0.9735356791338584,
+        "pass@4": 0.9843826020809897,
+        "pass@8": 0.9903981612161823,
+        "pass@16": 0.9936567666222805,
+        "pass@32": 0.9955643444237633,
+        "pass@64": 0.9966508344175822,
+        "pass@128": 0.997
+      },
+      "16": {
+        "pass@1": 0.8934609375,
+        "pass@2": 0.9216697834645677,
+        "pass@4": 0.9393977767154105,
+        "pass@8": 0.9517931687352534,
+        "pass@16": 0.96139777462629,
+        "pass@32": 0.9693210349487325,
+        "pass@64": 0.9751328706018372,
+        "pass@128": 0.979
+      },
+      "17": {
+        "pass@1": 0.8661875,
+        "pass@2": 0.9038288631889774,
+        "pass@4": 0.9277408284589423,
+        "pass@8": 0.9448044024114316,
+        "pass@16": 0.9568565852772714,
+        "pass@32": 0.9646994651551266,
+        "pass@64": 0.9694077197479471,
+        "pass@128": 0.972
+      },
+      "18": {
+        "pass@1": 0.788859375,
+        "pass@2": 0.8561178641732294,
+        "pass@4": 0.8988476714473197,
+        "pass@8": 0.9267583655390567,
+        "pass@16": 0.9448361294806787,
+        "pass@32": 0.9568568515114735,
+        "pass@64": 0.9657552890983766,
+        "pass@128": 0.973
+      },
+      "19": {
+        "pass@1": 0.625640625,
+        "pass@2": 0.7169927411417328,
+        "pass@4": 0.7811060572740908,
+        "pass@8": 0.8285917496219818,
+        "pass@16": 0.8654698909087529,
+        "pass@32": 0.8941006950628526,
+        "pass@64": 0.9155477279248722,
+        "pass@128": 0.932
+      },
+      "20": {
+        "pass@1": 0.4891484375,
+        "pass@2": 0.5897840797244098,
+        "pass@4": 0.6696891319835014,
+        "pass@8": 0.7328330903927104,
+        "pass@16": 0.7830265616151493,
+        "pass@32": 0.824425487387794,
+        "pass@64": 0.8589137622794993,
+        "pass@128": 0.886
+      }
+    },
+    "per_op_accuracy": {
+      "10": 0.9546875,
+      "2": 1.0,
+      "3": 0.99915625,
+      "4": 0.99971875,
+      "5": 0.9941953125,
+      "6": 0.99975,
+      "7": 0.9958359375,
+      "8": 0.9950703125,
+      "9": 0.99559375,
+      "11": 0.9726171875,
+      "12": 0.9644375,
+      "13": 0.94846875,
+      "14": 0.9449453125,
+      "15": 0.952953125,
+      "16": 0.8934609375,
+      "17": 0.8661875,
+      "18": 0.788859375,
+      "19": 0.625640625,
+      "20": 0.4891484375
+    },
+    "per_op_avg_response_len": {
+      "10": 281.6347734375,
+      "2": 115.6326015625,
+      "3": 144.829234375,
+      "4": 159.923921875,
+      "5": 173.2866484375,
+      "6": 198.9441953125,
+      "7": 224.1663515625,
+      "8": 243.7276171875,
+      "9": 259.660875,
+      "11": 300.8103125,
+      "12": 313.388375,
+      "13": 334.3829921875,
+      "14": 361.2994765625,
+      "15": 381.435625,
+      "16": 390.2542265625,
+      "17": 410.935984375,
+      "18": 425.279296875,
+      "19": 422.265296875,
+      "20": 430.9158984375
+    },
+    "per_op_avg_loss": {},
+    "per_op_length": {
+      "10": 281.6347734375,
+      "2": 115.6326015625,
+      "3": 144.829234375,
+      "4": 159.923921875,
+      "5": 173.2866484375,
+      "6": 198.9441953125,
+      "7": 224.1663515625,
+      "8": 243.7276171875,
+      "9": 259.660875,
+      "11": 300.8103125,
+      "12": 313.388375,
+      "13": 334.3829921875,
+      "14": 361.2994765625,
+      "15": 381.435625,
+      "16": 390.2542265625,
+      "17": 410.935984375,
+      "18": 425.279296875,
+      "19": 422.265296875,
+      "20": 430.9158984375
+    },
+    "per_op_loss": {},
+    "per_template": {
+      "crazy_zootopia": {
+        "count": 809728,
+        "correct": 738018,
+        "answer_accuracy": 0.9114393969332911,
+        "avg_response_len": 280.1749056473285,
+        "resp_tokens_sum": 226865466,
+        "pass_at_k": {
+          "pass@1": 0.9114393969332911,
+          "pass@2": 0.9346978341788974,
+          "pass@4": 0.9510085321100552,
+          "pass@8": 0.9624914679892927,
+          "pass@16": 0.9708110810206952,
+          "pass@32": 0.9770193310184669,
+          "pass@64": 0.981604927339997,
+          "pass@128": 0.9851406892190958
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9615366887905604,
+            "pass@2": 0.970145808654449,
+            "pass@4": 0.9771259859110534,
+            "pass@8": 0.9837142589656627,
+            "pass@16": 0.9897806295670789,
+            "pass@32": 0.9934434017257641,
+            "pass@64": 0.9955847044260153,
+            "pass@128": 0.9970501474926253
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9997610856269113,
+            "pass@2": 0.9999864552481399,
+            "pass@4": 0.9999999638806617,
+            "pass@8": 0.9999999999999809,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.99931640625,
+            "pass@2": 0.9999350239911416,
+            "pass@4": 0.9999994973284589,
+            "pass@8": 0.999999999990623,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9954861111111111,
+            "pass@2": 0.9972468285214349,
+            "pass@4": 0.9985486585010207,
+            "pass@8": 0.9994671341886722,
+            "pass@16": 0.9999236250321564,
+            "pass@32": 0.9999990493973,
+            "pass@64": 0.9999999999949286,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9995555763473054,
+            "pass@2": 0.9999499033429206,
+            "pass@4": 0.9999993320445721,
+            "pass@8": 0.9999999999490911,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9964404585798816,
+            "pass@2": 0.9978735207100592,
+            "pass@4": 0.9986706386183974,
+            "pass@8": 0.9994230191252826,
+            "pass@16": 0.9999007629980622,
+            "pass@32": 0.9999981593967747,
+            "pass@64": 0.99999999996853,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.997245592948718,
+            "pass@2": 0.9984009848828992,
+            "pass@4": 0.9992322474113812,
+            "pass@8": 0.9998262245762686,
+            "pass@16": 0.9999926289511722,
+            "pass@32": 0.9999999947902106,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9950704225352113,
+            "pass@2": 0.9983818759010756,
+            "pass@4": 0.999779442534472,
+            "pass@8": 0.9999935759719568,
+            "pass@16": 0.9999999938621692,
+            "pass@32": 0.9999999999999997,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9698123229461756,
+            "pass@2": 0.981584311079387,
+            "pass@4": 0.9891040887948495,
+            "pass@8": 0.9937797636155625,
+            "pass@16": 0.9970789483598115,
+            "pass@32": 0.999223562425918,
+            "pass@64": 0.9999570534856826,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9659030720338984,
+            "pass@2": 0.9809874632990792,
+            "pass@4": 0.9906215364498931,
+            "pass@8": 0.9956137072056146,
+            "pass@16": 0.9978803736651605,
+            "pass@32": 0.9993246177441057,
+            "pass@64": 0.9999568688255516,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9498922413793104,
+            "pass@2": 0.9708758608347934,
+            "pass@4": 0.9840897281060871,
+            "pass@8": 0.9909108589255634,
+            "pass@16": 0.9943999521503089,
+            "pass@32": 0.9963005415376021,
+            "pass@64": 0.9980499904733999,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9233839563862928,
+            "pass@2": 0.944520460789364,
+            "pass@4": 0.9611923819102052,
+            "pass@8": 0.9751079976868544,
+            "pass@16": 0.9861293435926446,
+            "pass@32": 0.9935272719026856,
+            "pass@64": 0.9975030863628863,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.9595375722543352,
+            "pass@2": 0.9784015719584908,
+            "pass@4": 0.987961396353635,
+            "pass@8": 0.9923778119086278,
+            "pass@16": 0.9938310119401556,
+            "pass@32": 0.9941908530213401,
+            "pass@64": 0.9942196182961773,
+            "pass@128": 0.9942196531791907
+          },
+          "16": {
+            "pass@1": 0.8793850806451613,
+            "pass@2": 0.9133687611125221,
+            "pass@4": 0.9350571322737883,
+            "pass@8": 0.9489252481008612,
+            "pass@16": 0.9594660322806725,
+            "pass@32": 0.9677192403291304,
+            "pass@64": 0.9722136540329196,
+            "pass@128": 0.9741935483870968
+          },
+          "17": {
+            "pass@1": 0.8896992413294798,
+            "pass@2": 0.9255197191752769,
+            "pass@4": 0.9464512051600483,
+            "pass@8": 0.9595738090472379,
+            "pass@16": 0.9673121876199731,
+            "pass@32": 0.9725731938539892,
+            "pass@64": 0.9768482847119901,
+            "pass@128": 0.9797687861271677
+          },
+          "18": {
+            "pass@1": 0.7444419642857143,
+            "pass@2": 0.8136916479190104,
+            "pass@4": 0.8625618206652741,
+            "pass@8": 0.8958082898451949,
+            "pass@16": 0.9174575405950798,
+            "pass@32": 0.9337781303810576,
+            "pass@64": 0.947397000627292,
+            "pass@128": 0.9571428571428572
+          },
+          "19": {
+            "pass@1": 0.6253753753753754,
+            "pass@2": 0.7184758577475104,
+            "pass@4": 0.7833039738275956,
+            "pass@8": 0.8296772377534075,
+            "pass@16": 0.8660549147719705,
+            "pass@32": 0.8940814351693145,
+            "pass@64": 0.9133406551952218,
+            "pass@128": 0.9279279279279279
+          },
+          "20": {
+            "pass@1": 0.4760932807807808,
+            "pass@2": 0.5768932071835613,
+            "pass@4": 0.6601997517765237,
+            "pass@8": 0.7265525943478015,
+            "pass@16": 0.7789762742387952,
+            "pass@32": 0.8215439955324653,
+            "pass@64": 0.8572865791166149,
+            "pass@128": 0.8888888888888888
+          }
+        }
+      },
+      "teachers_in_school": {
+        "count": 821120,
+        "correct": 753428,
+        "answer_accuracy": 0.9175613795791114,
+        "avg_response_len": 289.52658198558066,
+        "resp_tokens_sum": 237736067,
+        "pass_at_k": {
+          "pass@1": 0.9175613795791114,
+          "pass@2": 0.9393609304901669,
+          "pass@4": 0.9544253548538294,
+          "pass@8": 0.9651838128249977,
+          "pass@16": 0.9731036723828017,
+          "pass@32": 0.9792285383144879,
+          "pass@64": 0.9839183408625037,
+          "pass@128": 0.987373343725643
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9446654524539877,
+            "pass@2": 0.9550259498333415,
+            "pass@4": 0.9634138444542589,
+            "pass@8": 0.970866877957459,
+            "pass@16": 0.9782378206275941,
+            "pass@32": 0.9857900637806933,
+            "pass@64": 0.9923585282128631,
+            "pass@128": 0.9969325153374233
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9988873106060606,
+            "pass@2": 0.9997856269386781,
+            "pass@4": 0.9999895822681255,
+            "pass@8": 0.9999999777033803,
+            "pass@16": 0.9999999999999803,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.9999537721893491,
+            "pass@2": 0.999999636001491,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9965579710144927,
+            "pass@2": 0.9972633515919205,
+            "pass@4": 0.9975327839454851,
+            "pass@8": 0.9979120550709301,
+            "pass@16": 0.9985302315822023,
+            "pass@32": 0.9993303337082865,
+            "pass@64": 0.9999164669633687,
+            "pass@128": 1.0
+          },
+          "6": {
+            "pass@1": 0.9997313753581661,
+            "pass@2": 0.9999915393813595,
+            "pass@4": 0.9999999905993125,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9975162572254336,
+            "pass@2": 0.9985822931364069,
+            "pass@4": 0.9993384239181086,
+            "pass@8": 0.9998572295938166,
+            "pass@16": 0.9999945349140869,
+            "pass@32": 0.9999999969725001,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9966769366197183,
+            "pass@2": 0.9978297937229678,
+            "pass@4": 0.9984739589769589,
+            "pass@8": 0.9991927685739189,
+            "pass@16": 0.9997893254704586,
+            "pass@32": 0.999989840105361,
+            "pass@64": 0.9999999973246758,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9973100673652695,
+            "pass@2": 0.9978701553585743,
+            "pass@4": 0.9984959510113631,
+            "pass@8": 0.9992628054872507,
+            "pass@16": 0.9998367378399854,
+            "pass@32": 0.9999945907477397,
+            "pass@64": 0.9999999995090815,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9770566641566265,
+            "pass@2": 0.9841648829570252,
+            "pass@4": 0.989164249235412,
+            "pass@8": 0.9922976666641117,
+            "pass@16": 0.9941890352635745,
+            "pass@32": 0.9952655846372421,
+            "pass@64": 0.9962408325530068,
+            "pass@128": 0.9969879518072289
+          },
+          "12": {
+            "pass@1": 0.9620640673981191,
+            "pass@2": 0.9773606620097254,
+            "pass@4": 0.9866140545127782,
+            "pass@8": 0.9920244775588233,
+            "pass@16": 0.9951791105256566,
+            "pass@32": 0.9972397387226369,
+            "pass@64": 0.9990357602586,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9582020308123249,
+            "pass@2": 0.9730195720020293,
+            "pass@4": 0.9829933805543213,
+            "pass@8": 0.9895737184859488,
+            "pass@16": 0.9944316023575817,
+            "pass@32": 0.9979028151511814,
+            "pass@64": 0.9996096734768417,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9545869883040936,
+            "pass@2": 0.9759829569001243,
+            "pass@4": 0.9895990122945156,
+            "pass@8": 0.9958761100329401,
+            "pass@16": 0.9982192698442767,
+            "pass@32": 0.9995253069447078,
+            "pass@64": 0.9999798903618302,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.964654356060606,
+            "pass@2": 0.9848753280839894,
+            "pass@4": 0.994307931394939,
+            "pass@8": 0.9984355730943542,
+            "pass@16": 0.9997865737709953,
+            "pass@32": 0.999996225078149,
+            "pass@64": 0.9999999999355342,
+            "pass@128": 1.0
+          },
+          "16": {
+            "pass@1": 0.9025106837606838,
+            "pass@2": 0.9249840865244409,
+            "pass@4": 0.9394740470475377,
+            "pass@8": 0.951603221594124,
+            "pass@16": 0.9618929894371758,
+            "pass@32": 0.9704062836352245,
+            "pass@64": 0.9772483460299233,
+            "pass@128": 0.9829059829059829
+          },
+          "17": {
+            "pass@1": 0.8692498059006211,
+            "pass@2": 0.9090858377757125,
+            "pass@4": 0.9353135909447651,
+            "pass@8": 0.9535516185363221,
+            "pass@16": 0.965466357509199,
+            "pass@32": 0.9717789256494813,
+            "pass@64": 0.9742932379404277,
+            "pass@128": 0.9751552795031055
+          },
+          "18": {
+            "pass@1": 0.8068244485294118,
+            "pass@2": 0.8750351001621121,
+            "pass@4": 0.9166487397714989,
+            "pass@8": 0.9411302798270424,
+            "pass@16": 0.9559175964520414,
+            "pass@32": 0.9659157855028336,
+            "pass@64": 0.9733500817730232,
+            "pass@128": 0.9794117647058823
+          },
+          "19": {
+            "pass@1": 0.6194428943452381,
+            "pass@2": 0.7078185051087363,
+            "pass@4": 0.7742094414537468,
+            "pass@8": 0.8272903079106774,
+            "pass@16": 0.8697325655416732,
+            "pass@32": 0.9032315125581434,
+            "pass@64": 0.9280735994123079,
+            "pass@128": 0.9464285714285714
+          },
+          "20": {
+            "pass@1": 0.47754491017964074,
+            "pass@2": 0.5815400449101793,
+            "pass@4": 0.6627141611041132,
+            "pass@8": 0.7251319869091699,
+            "pass@16": 0.7741932335037112,
+            "pass@32": 0.8160539579293532,
+            "pass@64": 0.8519449321090409,
+            "pass@128": 0.8802395209580839
+          }
+        }
+      },
+      "movie_festival_awards": {
+        "count": 801152,
+        "correct": 733287,
+        "answer_accuracy": 0.9152907313468606,
+        "avg_response_len": 310.44483568661127,
+        "resp_tokens_sum": 248713501,
+        "pass_at_k": {
+          "pass@1": 0.9152907313468606,
+          "pass@2": 0.9382467396555126,
+          "pass@4": 0.9536234835709834,
+          "pass@8": 0.9645607797964179,
+          "pass@16": 0.9724503971693556,
+          "pass@32": 0.9781535877149617,
+          "pass@64": 0.9824174417387895,
+          "pass@128": 0.9854609362517974
+        },
+        "per_op_pass_at_k": {
+          "10": {
+            "pass@1": 0.9575093283582089,
+            "pass@2": 0.9675553090844993,
+            "pass@4": 0.9764339019189767,
+            "pass@8": 0.9840698697287039,
+            "pass@16": 0.9892746985021801,
+            "pass@32": 0.9922997890645133,
+            "pass@64": 0.9946955387643621,
+            "pass@128": 0.9970149253731343
+          },
+          "2": {
+            "pass@1": 1.0,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "3": {
+            "pass@1": 0.9988383746355685,
+            "pass@2": 0.9995426671793577,
+            "pass@4": 0.9999317049654508,
+            "pass@8": 0.999998701511788,
+            "pass@16": 0.9999999997759129,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "4": {
+            "pass@1": 0.9998629385964912,
+            "pass@2": 0.9999964025878345,
+            "pass@4": 0.9999999986295574,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "5": {
+            "pass@1": 0.9906020220588235,
+            "pass@2": 0.9926593619731356,
+            "pass@4": 0.9945639799620636,
+            "pass@8": 0.9960446992193006,
+            "pass@16": 0.9968320635603195,
+            "pass@32": 0.997048210623365,
+            "pass@64": 0.9970588207360587,
+            "pass@128": 0.9970588235294118
+          },
+          "6": {
+            "pass@1": 0.9999753548895899,
+            "pass@2": 1.0,
+            "pass@4": 1.0,
+            "pass@8": 1.0,
+            "pass@16": 1.0,
+            "pass@32": 1.0,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "7": {
+            "pass@1": 0.9933494857594937,
+            "pass@2": 0.9949833518140139,
+            "pass@4": 0.9964600671949234,
+            "pass@8": 0.9976420959829561,
+            "pass@16": 0.9985911723443348,
+            "pass@32": 0.9994590693089006,
+            "pass@64": 0.9999562541175095,
+            "pass@128": 1.0
+          },
+          "8": {
+            "pass@1": 0.9913194444444444,
+            "pass@2": 0.9966260551890473,
+            "pass@4": 0.9991402473902474,
+            "pass@8": 0.9999150306942846,
+            "pass@16": 0.9999991362232366,
+            "pass@32": 0.9999999999722541,
+            "pass@64": 1.0,
+            "pass@128": 1.0
+          },
+          "9": {
+            "pass@1": 0.9943478697749196,
+            "pass@2": 0.9973257462592097,
+            "pass@4": 0.9988976594967428,
+            "pass@8": 0.9997409379688414,
+            "pass@16": 0.9999841784174797,
+            "pass@32": 0.999999971428724,
+            "pass@64": 0.9999999999999997,
+            "pass@128": 1.0
+          },
+          "11": {
+            "pass@1": 0.9710813492063493,
+            "pass@2": 0.9827689507561553,
+            "pass@4": 0.9910190773772323,
+            "pass@8": 0.9964069864008952,
+            "pass@16": 0.9991975166401181,
+            "pass@32": 0.9999605809858803,
+            "pass@64": 0.9999999808615654,
+            "pass@128": 1.0
+          },
+          "12": {
+            "pass@1": 0.9651662844036697,
+            "pass@2": 0.9830220297864142,
+            "pass@4": 0.9943498286911382,
+            "pass@8": 0.9989944241978972,
+            "pass@16": 0.9999475847370622,
+            "pass@32": 0.9999998481010195,
+            "pass@64": 0.9999999999999678,
+            "pass@128": 1.0
+          },
+          "13": {
+            "pass@1": 0.9363425925925926,
+            "pass@2": 0.9636441929133857,
+            "pass@4": 0.9812329428150182,
+            "pass@8": 0.9909109265469578,
+            "pass@16": 0.995563713627039,
+            "pass@32": 0.997463540610219,
+            "pass@64": 0.9984545373712423,
+            "pass@128": 1.0
+          },
+          "14": {
+            "pass@1": 0.9556982566765578,
+            "pass@2": 0.971536645131896,
+            "pass@4": 0.9815543184142042,
+            "pass@8": 0.9879400706835424,
+            "pass@16": 0.9919984009149101,
+            "pass@32": 0.9955077699507003,
+            "pass@64": 0.9984439818222332,
+            "pass@128": 1.0
+          },
+          "15": {
+            "pass@1": 0.9340036651234568,
+            "pass@2": 0.9567897437299503,
+            "pass@4": 0.9704516715503154,
+            "pass@8": 0.9800978369588287,
+            "pass@16": 0.9872273677362894,
+            "pass@32": 0.9925171450697239,
+            "pass@64": 0.9958359460135142,
+            "pass@128": 0.9969135802469136
+          },
+          "16": {
+            "pass@1": 0.8969625737463127,
+            "pass@2": 0.9258290653380717,
+            "pass@4": 0.9432881274243372,
+            "pass@8": 0.9546124219600274,
+            "pass@16": 0.9626515201381516,
+            "pass@32": 0.9696621383207606,
+            "pass@64": 0.9756120011655725,
+            "pass@128": 0.9793510324483776
+          },
+          "17": {
+            "pass@1": 0.8387142319277109,
+            "pass@2": 0.8761247005739499,
+            "pass@4": 0.9008967927390097,
+            "pass@8": 0.9209284437120231,
+            "pass@16": 0.937609642297589,
+            "pass@32": 0.9496274880196173,
+            "pass@64": 0.9569150319903035,
+            "pass@128": 0.9608433734939759
+          },
+          "18": {
+            "pass@1": 0.819304435483871,
+            "pass@2": 0.8832704946659896,
+            "pass@4": 0.9202918151360103,
+            "pass@8": 0.9459392546840153,
+            "pass@16": 0.9635935725119584,
+            "pass@32": 0.9729778671198072,
+            "pass@64": 0.9781526163741832,
+            "pass@128": 0.9838709677419355
+          },
+          "19": {
+            "pass@1": 0.6321988293051359,
+            "pass@2": 0.7248134827651832,
+            "pass@4": 0.7858956545650826,
+            "pass@8": 0.828820803601541,
+            "pass@16": 0.8605542666697465,
+            "pass@32": 0.8848513260783547,
+            "pass@64": 0.905053052393738,
+            "pass@128": 0.9214501510574018
+          },
+          "20": {
+            "pass@1": 0.5138419669669669,
+            "pass@2": 0.6109437439407909,
+            "pass@4": 0.6861744289283656,
+            "pass@8": 0.7468378163580444,
+            "pass@16": 0.7959367036137914,
+            "pass@32": 0.8357036484296656,
+            "pass@64": 0.8675307030307697,
+            "pass@128": 0.8888888888888888
+          }
+        }
+      }
+    }
+  }
+}

Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_ood_generations.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb18d559562580a79812b126ac3310f0999c5cc942f0954d4c7b557696c292e3
+size 2708951048

Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_id_generations.jsonl.graph_results.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7ab4985f208a74ba69c0cafcd566c44396812b21d93da2410cc3bacad99f1ab
+size 373215241

Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_ood_generations.jsonl.graph_results.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e8a42cbbf5bba96958a9c3cf17f5ddbfc0a59a3aa61c3da10ef4fa15cd02490
+size 423543596

Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/composition_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M-midtrain-diff_ar-block8/composition", "results_dir": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs", "per_file": [{"file": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024156, "value_mismatch": 9626, "dependency_mismatch": 126467, "answer_mismatch": 8447, "extra_nodes": 11789, "missing_nodes": 9372, "perfect_with_extra": 832, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337828, "dependency_mismatch": 40795, "value_mismatch": 2837, "answer_mismatch": 2396, "extra_nodes": 3565, "missing_nodes": 2829, "perfect_with_extra": 194}, "teachers_in_school": {"total": 390656, "perfect": 346878, "value_mismatch": 3053, "dependency_mismatch": 43324, "answer_mismatch": 2898, "extra_nodes": 3875, "perfect_with_extra": 245, "missing_nodes": 3417}, "movie_festival_awards": {"total": 382336, "perfect": 339450, "dependency_mismatch": 42348, "missing_nodes": 3126, "extra_nodes": 4349, "perfect_with_extra": 393, "value_mismatch": 3736, "answer_mismatch": 3153}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/checkpoint-38645_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 619058, "value_mismatch": 253560, "dependency_mismatch": 638256, "answer_mismatch": 198821, "extra_nodes": 43329, "missing_nodes": 193491, "perfect_with_extra": 424, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "perfect": 206999, "extra_nodes": 16615, "perfect_with_extra": 50, "value_mismatch": 87251, "dependency_mismatch": 215764, "answer_mismatch": 69314, "missing_nodes": 70355}, "movie_festival_awards": {"total": 418816, "perfect": 204316, "value_mismatch": 82684, "dependency_mismatch": 206999, "answer_mismatch": 64712, "extra_nodes": 14071, "missing_nodes": 59687, "perfect_with_extra": 220}, "teachers_in_school": {"total": 430464, "perfect": 207743, "value_mismatch": 83625, "dependency_mismatch": 215493, "answer_mismatch": 64795, "missing_nodes": 63449, "extra_nodes": 12643, "perfect_with_extra": 154}}, "results_path": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/checkpoint-38645_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1643214, "value_mismatch": 263186, "dependency_mismatch": 764723, "answer_mismatch": 207268, "extra_nodes": 55118, "missing_nodes": 202863, "perfect_with_extra": 1256, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M-midtrain-diff_ar-block8/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 544827, "dependency_mismatch": 256559, "value_mismatch": 90088, "answer_mismatch": 71710, "extra_nodes": 20180, "missing_nodes": 73184, "perfect_with_extra": 244}, "teachers_in_school": {"total": 821120, "perfect": 554621, "value_mismatch": 86678, "dependency_mismatch": 258817, "answer_mismatch": 67693, "extra_nodes": 16518, "perfect_with_extra": 399, "missing_nodes": 66866}, "movie_festival_awards": {"total": 801152, "perfect": 543766, "dependency_mismatch": 249347, "missing_nodes": 62813, "extra_nodes": 18420, "perfect_with_extra": 613, "value_mismatch": 86420, "answer_mismatch": 67865}}}}