{ "rows": { "test_vsr": { "acc": { "metric": "accuracy", "questions": 902, "pictures": 547, "a": 0.6607538802660754, "b": 0.7305986696230599, "difference": 0.06984478935698446, "ci95": [ 0.03618421052631582, 0.10647639956092214 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 902, "pictures": 547, "a": 0.6155837500135749, "b": 0.5904692457609072, "difference": -0.025114504252667702, "ci95": [ -0.06730200801082653, 0.01835022339389425 ], "resamples": 2000 } }, "test_v7w": { "acc": { "metric": "accuracy", "questions": 1000, "pictures": 1000, "a": 0.711, "b": 0.783, "difference": 0.07200000000000006, "ci95": [ 0.04800000000000004, 0.09600000000000009 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1000, "pictures": 1000, "a": 0.7061062017078975, "b": 0.5966842790244825, "difference": -0.109421922683415, "ci95": [ -0.14233793828593067, -0.07693998792887025 ], "resamples": 2000 } }, "bench_pope": { "acc": { "metric": "accuracy", "questions": 9000, "pictures": 500, "a": 0.8481111111111111, "b": 0.8653333333333333, "difference": 0.01722222222222214, "ci95": [ 0.00855555555555565, 0.02577777777777779 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 9000, "pictures": 500, "a": 0.37213111549634703, "b": 0.36302066313255416, "difference": -0.009110452363792876, "ci95": [ -0.02169886047608327, 0.003435790059213739 ], "resamples": 2000 } }, "test_vqa_yesno": { "acc": { "metric": "accuracy", "questions": 1000, "pictures": 977, "a": 0.683, "b": 0.722, "difference": 0.038999999999999924, "ci95": [ 0.015090543259557276, 0.06480558325024932 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1000, "pictures": 977, "a": 0.5941348191891218, "b": 0.571303429093119, "difference": -0.022831390096002768, "ci95": [ -0.04409385128971266, -0.0019899478295204442 ], "resamples": 2000 } }, "test_vqa_choice": { "acc": { "metric": "accuracy", "questions": 1453, "pictures": 1394, "a": 0.8823124569855472, "b": 0.9050240880935995, "difference": 0.0227116311080523, "ci95": [ 0.007686932215234088, 0.03782668500687758 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1453, "pictures": 1394, "a": 0.2975529214462992, "b": 0.28024055404603077, "difference": -0.017312367400268425, "ci95": [ -0.04801307597139293, 0.017763019358089716 ], "resamples": 2000 } }, "test_gqa": { "acc": { "metric": "accuracy", "questions": 986, "pictures": 880, "a": 0.781947261663286, "b": 0.8427991886409736, "difference": 0.0608519269776876, "ci95": [ 0.03567787971457692, 0.08611955420466055 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 986, "pictures": 880, "a": 0.4363967446492391, "b": 0.38284475493705167, "difference": -0.05355198971218744, "ci95": [ -0.08716986207647759, -0.02041387915707188 ], "resamples": 2000 } }, "test_exist": { "acc": { "metric": "accuracy", "questions": 1120, "pictures": 1104, "a": 0.9392857142857143, "b": 0.9553571428571429, "difference": 0.016071428571428625, "ci95": [ 0.0026929982046678402, 0.02930728241563063 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1120, "pictures": 1104, "a": 0.16983095379080976, "b": 0.19567675111207672, "difference": 0.02584579732126696, "ci95": [ 0.007774706565661066, 0.04337850870878224 ], "resamples": 2000 } }, "test_position": { "acc": { "metric": "accuracy", "questions": 1276, "pictures": 1232, "a": 0.8863636363636364, "b": 0.9380877742946708, "difference": 0.051724137931034475, "ci95": [ 0.03299293008641002, 0.07092198581560283 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1276, "pictures": 1232, "a": 0.27279601591009106, "b": 0.23840018591039677, "difference": -0.03439582999969429, "ci95": [ -0.08219989842410494, 0.015482905705434202 ], "resamples": 2000 } }, "test_relation": { "acc": { "metric": "accuracy", "questions": 2366, "pictures": 1768, "a": 0.7489433643279797, "b": 0.9167371090448013, "difference": 0.16779374471682162, "ci95": [ 0.1491782553729456, 0.18608549874266556 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2366, "pictures": 1768, "a": 0.4909602253543323, "b": 0.2805283450848128, "difference": -0.21043188026951953, "ci95": [ -0.25335914427956835, -0.16455708571446082 ], "resamples": 2000 } }, "test_size": { "acc": { "metric": "accuracy", "questions": 1198, "pictures": 1081, "a": 0.8564273789649416, "b": 0.8697829716193656, "difference": 0.013355592654423987, "ci95": [ -0.002523128679562725, 0.030252100840336138 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1198, "pictures": 1081, "a": 0.35047560484582363, "b": 0.5065374303761236, "difference": 0.1560618255303, "ci95": [ 0.12203261708552204, 0.18841077280205148 ], "resamples": 2000 } }, "test_position_flip": { "acc": { "metric": "accuracy", "questions": 640, "pictures": 636, "a": 0.828125, "b": 0.9328125, "difference": 0.10468750000000004, "ci95": [ 0.07931570762052875, 0.1316614420062696 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 640, "pictures": 636, "a": 0.3792944504698811, "b": 0.28297517066909245, "difference": -0.09631927980078864, "ci95": [ -0.16729839354099188, -0.019257792204828483 ], "resamples": 2000 } }, "test_relation_flip": { "acc": { "metric": "accuracy", "questions": 1786, "pictures": 1460, "a": 0.7021276595744681, "b": 0.9143337066069429, "difference": 0.2122060470324748, "ci95": [ 0.18990246701090074, 0.2346089850249583 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1786, "pictures": 1460, "a": 0.5234402751432231, "b": 0.3004006765189471, "difference": -0.22303959862427603, "ci95": [ -0.27563722667679236, -0.16170787803919912 ], "resamples": 2000 } }, "bench_lv_vqav2_yesno": { "acc": { "metric": "accuracy", "questions": 4887, "pictures": 2069, "a": 0.7249846531614488, "b": 0.7558829547779824, "difference": 0.03089830161653362, "ci95": [ 0.01905156347069792, 0.04241915161696774 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 4887, "pictures": 2069, "a": 0.551730515857613, "b": 0.5256937856914787, "difference": -0.02603673016613428, "ci95": [ -0.03484135061656102, -0.01729670636981273 ], "resamples": 2000 } }, "bench_lv_aokvqa": { "acc": { "metric": "accuracy", "questions": 1138, "pictures": 1138, "a": 0.6124780316344464, "b": 0.6335676625659051, "difference": 0.02108963093145877, "ci95": [ -0.0035149384885764245, 0.0448154657293498 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1138, "pictures": 1138, "a": 0.9357384620349536, "b": 0.9235836608133525, "difference": -0.012154801221601064, "ci95": [ -0.050025256755278025, 0.02747038629963383 ], "resamples": 2000 } }, "bench_lv_scienceqa": { "acc": { "metric": "accuracy", "questions": 2097, "pictures": 2097, "a": 0.7892226990939437, "b": 0.8206962327134001, "difference": 0.03147353361945637, "ci95": [ 0.015736766809728242, 0.047687172150691515 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2097, "pictures": 2097, "a": 0.47098928326742956, "b": 0.46387570466731803, "difference": -0.007113578600111525, "ci95": [ -0.036043017961029544, 0.02201631592471548 ], "resamples": 2000 } }, "bench_whatsup": { "acc": { "metric": "accuracy", "questions": 820, "pictures": 820, "a": 0.2865853658536585, "b": 0.46707317073170734, "difference": 0.18048780487804883, "ci95": [ 0.1353658536585366, 0.22073170731707314 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 820, "pictures": 820, "a": 1.490915758604817, "b": 1.2141925113554008, "difference": -0.2767232472494161, "ci95": [ -0.335244532597462, -0.219079539978984 ], "resamples": 2000 } }, "bench_countbench": { "acc": { "metric": "accuracy", "questions": 491, "pictures": 491, "a": 0.3564154786150713, "b": 0.5010183299389002, "difference": 0.14460285132382888, "ci95": [ 0.09368635437881867, 0.1955193482688391 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 491, "pictures": 491, "a": 1.6021549698788784, "b": 1.3872406640068242, "difference": -0.21491430587205418, "ci95": [ -0.2562705789097375, -0.17616865330760545 ], "resamples": 2000 } }, "bench_cogent_b": { "acc": { "metric": "accuracy", "questions": 2000, "pictures": 2000, "a": 0.5775, "b": 0.618, "difference": 0.04049999999999998, "ci95": [ 0.01649999999999996, 0.0655 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2000, "pictures": 2000, "a": 0.8426200623459391, "b": 0.8368894876234722, "difference": -0.005730574722466963, "ci95": [ -0.02600691985620429, 0.013703787042339766 ], "resamples": 2000 } }, "test_count_coco": { "acc": { "metric": "accuracy", "questions": 1100, "pictures": 1100, "a": 0.30636363636363634, "b": 0.34454545454545454, "difference": 0.038181818181818206, "ci95": [ 0.011818181818181783, 0.06545454545454549 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1100, "pictures": 1100, "a": 1.9893882985025666, "b": 1.779380167228201, "difference": -0.21000813127436557, "ci95": [ -0.25964636612728054, -0.1622590152983625 ], "resamples": 2000 } }, "bench_mmbench": { "acc": { "metric": "accuracy", "questions": 4294, "pictures": 4294, "a": 0.6040987424312995, "b": 0.6266884024219842, "difference": 0.02258965999068474, "ci95": [ 0.011178388448998544, 0.03446669771774569 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 4294, "pictures": 4294, "a": 0.9621266293403985, "b": 0.9076105848749486, "difference": -0.05451604446544989, "ci95": [ -0.07615708256293019, -0.03108140813878635 ], "resamples": 2000 } }, "bench_cvbench": { "acc": { "metric": "accuracy", "questions": 2638, "pictures": 2638, "a": 0.5443517816527672, "b": 0.5511751326762699, "difference": 0.006823351023502733, "ci95": [ -0.009855951478392688, 0.02501895375284313 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2638, "pictures": 2638, "a": 0.9016688344057614, "b": 0.8083227674466051, "difference": -0.09334606695915626, "ci95": [ -0.11244997195903339, -0.07520308890416327 ], "resamples": 2000 } }, "bench_mme": { "acc": { "metric": "accuracy", "questions": 2374, "pictures": 1187, "a": 0.6533277169334457, "b": 0.6474304970513901, "difference": -0.005897219882055604, "ci95": [ -0.02064026958719467, 0.008845829823083351 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2374, "pictures": 1187, "a": 0.6057342796352463, "b": 0.6429894737804899, "difference": 0.037255194145243586, "ci95": [ 0.023705705678477296, 0.05017788218628971 ], "resamples": 2000 } }, "bench_naturalbench": { "acc": { "metric": "accuracy", "questions": 7570, "pictures": 3800, "a": 0.6206076618229854, "b": 0.6240422721268164, "difference": 0.003434610303830965, "ci95": [ -0.007000396248844298, 0.013199577613516311 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 7570, "pictures": 3800, "a": 0.6743440115208639, "b": 0.6925552586321518, "difference": 0.018211247111287965, "ci95": [ 0.008188317325104721, 0.029288391972986494 ], "resamples": 2000 } }, "dev_mix": { "acc": { "metric": "accuracy", "questions": 5178, "pictures": 3042, "a": 0.7576284279644651, "b": 0.790845886442642, "difference": 0.03321745847817692, "ci95": [ 0.022335794056407288, 0.04369682907965966 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 5178, "pictures": 3042, "a": 0.5432908524336165, "b": 0.5260876670578333, "difference": -0.017203185375783248, "ci95": [ -0.034758412199066724, 0.0017373233636809315 ], "resamples": 2000 } }, "dev_sqa": { "acc": { "metric": "accuracy", "questions": 2017, "pictures": 1799, "a": 0.8041646008924145, "b": 0.8463063956370848, "difference": 0.04214179474467028, "ci95": [ 0.024938875305623442, 0.05867970660146704 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2017, "pictures": 1799, "a": 0.45615675456359894, "b": 0.416725354374512, "difference": -0.03943140018908692, "ci95": [ -0.06878795828969625, -0.007295041475122277 ], "resamples": 2000 } } }, "verdict": { "missing": [], "1_no_set_worse": { "pass": true, "worse": [] }, "2_at_least_15_up": { "pass": true, "up": 22, "of": 23 }, "3_weak_4_of_5": { "pass": true, "by_ability": { "ScienceQA": true, "VSR": true, "counting": true, "A-OKVQA": false, "V7W": true } }, "4_nll_not_worse": { "pass": false, "worse": [ "test_exist", "test_size", "bench_mme" ] } } }