devision / evaluation /accept_vs_v0.3.json
lukbit's picture
deVision v0.4 (github model-v0.4, 0df9a73b091e0e8167f554da83a4034140354443)
3b1d1b4 verified
Raw History Blame Contribute Delete
14.9 kB
{
"rows": {
"test_vsr": {
"acc": {
"metric": "accuracy",
"questions": 902,
"pictures": 547,
"a": 0.6607538802660754,
"b": 0.7305986696230599,
"difference": 0.06984478935698446,
"ci95": [
0.03618421052631582,
0.10647639956092214
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 902,
"pictures": 547,
"a": 0.6155837500135749,
"b": 0.5904692457609072,
"difference": -0.025114504252667702,
"ci95": [
-0.06730200801082653,
0.01835022339389425
],
"resamples": 2000
}
},
"test_v7w": {
"acc": {
"metric": "accuracy",
"questions": 1000,
"pictures": 1000,
"a": 0.711,
"b": 0.783,
"difference": 0.07200000000000006,
"ci95": [
0.04800000000000004,
0.09600000000000009
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1000,
"pictures": 1000,
"a": 0.7061062017078975,
"b": 0.5966842790244825,
"difference": -0.109421922683415,
"ci95": [
-0.14233793828593067,
-0.07693998792887025
],
"resamples": 2000
}
},
"bench_pope": {
"acc": {
"metric": "accuracy",
"questions": 9000,
"pictures": 500,
"a": 0.8481111111111111,
"b": 0.8653333333333333,
"difference": 0.01722222222222214,
"ci95": [
0.00855555555555565,
0.02577777777777779
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 9000,
"pictures": 500,
"a": 0.37213111549634703,
"b": 0.36302066313255416,
"difference": -0.009110452363792876,
"ci95": [
-0.02169886047608327,
0.003435790059213739
],
"resamples": 2000
}
},
"test_vqa_yesno": {
"acc": {
"metric": "accuracy",
"questions": 1000,
"pictures": 977,
"a": 0.683,
"b": 0.722,
"difference": 0.038999999999999924,
"ci95": [
0.015090543259557276,
0.06480558325024932
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1000,
"pictures": 977,
"a": 0.5941348191891218,
"b": 0.571303429093119,
"difference": -0.022831390096002768,
"ci95": [
-0.04409385128971266,
-0.0019899478295204442
],
"resamples": 2000
}
},
"test_vqa_choice": {
"acc": {
"metric": "accuracy",
"questions": 1453,
"pictures": 1394,
"a": 0.8823124569855472,
"b": 0.9050240880935995,
"difference": 0.0227116311080523,
"ci95": [
0.007686932215234088,
0.03782668500687758
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1453,
"pictures": 1394,
"a": 0.2975529214462992,
"b": 0.28024055404603077,
"difference": -0.017312367400268425,
"ci95": [
-0.04801307597139293,
0.017763019358089716
],
"resamples": 2000
}
},
"test_gqa": {
"acc": {
"metric": "accuracy",
"questions": 986,
"pictures": 880,
"a": 0.781947261663286,
"b": 0.8427991886409736,
"difference": 0.0608519269776876,
"ci95": [
0.03567787971457692,
0.08611955420466055
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 986,
"pictures": 880,
"a": 0.4363967446492391,
"b": 0.38284475493705167,
"difference": -0.05355198971218744,
"ci95": [
-0.08716986207647759,
-0.02041387915707188
],
"resamples": 2000
}
},
"test_exist": {
"acc": {
"metric": "accuracy",
"questions": 1120,
"pictures": 1104,
"a": 0.9392857142857143,
"b": 0.9553571428571429,
"difference": 0.016071428571428625,
"ci95": [
0.0026929982046678402,
0.02930728241563063
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1120,
"pictures": 1104,
"a": 0.16983095379080976,
"b": 0.19567675111207672,
"difference": 0.02584579732126696,
"ci95": [
0.007774706565661066,
0.04337850870878224
],
"resamples": 2000
}
},
"test_position": {
"acc": {
"metric": "accuracy",
"questions": 1276,
"pictures": 1232,
"a": 0.8863636363636364,
"b": 0.9380877742946708,
"difference": 0.051724137931034475,
"ci95": [
0.03299293008641002,
0.07092198581560283
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1276,
"pictures": 1232,
"a": 0.27279601591009106,
"b": 0.23840018591039677,
"difference": -0.03439582999969429,
"ci95": [
-0.08219989842410494,
0.015482905705434202
],
"resamples": 2000
}
},
"test_relation": {
"acc": {
"metric": "accuracy",
"questions": 2366,
"pictures": 1768,
"a": 0.7489433643279797,
"b": 0.9167371090448013,
"difference": 0.16779374471682162,
"ci95": [
0.1491782553729456,
0.18608549874266556
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2366,
"pictures": 1768,
"a": 0.4909602253543323,
"b": 0.2805283450848128,
"difference": -0.21043188026951953,
"ci95": [
-0.25335914427956835,
-0.16455708571446082
],
"resamples": 2000
}
},
"test_size": {
"acc": {
"metric": "accuracy",
"questions": 1198,
"pictures": 1081,
"a": 0.8564273789649416,
"b": 0.8697829716193656,
"difference": 0.013355592654423987,
"ci95": [
-0.002523128679562725,
0.030252100840336138
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1198,
"pictures": 1081,
"a": 0.35047560484582363,
"b": 0.5065374303761236,
"difference": 0.1560618255303,
"ci95": [
0.12203261708552204,
0.18841077280205148
],
"resamples": 2000
}
},
"test_position_flip": {
"acc": {
"metric": "accuracy",
"questions": 640,
"pictures": 636,
"a": 0.828125,
"b": 0.9328125,
"difference": 0.10468750000000004,
"ci95": [
0.07931570762052875,
0.1316614420062696
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 640,
"pictures": 636,
"a": 0.3792944504698811,
"b": 0.28297517066909245,
"difference": -0.09631927980078864,
"ci95": [
-0.16729839354099188,
-0.019257792204828483
],
"resamples": 2000
}
},
"test_relation_flip": {
"acc": {
"metric": "accuracy",
"questions": 1786,
"pictures": 1460,
"a": 0.7021276595744681,
"b": 0.9143337066069429,
"difference": 0.2122060470324748,
"ci95": [
0.18990246701090074,
0.2346089850249583
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1786,
"pictures": 1460,
"a": 0.5234402751432231,
"b": 0.3004006765189471,
"difference": -0.22303959862427603,
"ci95": [
-0.27563722667679236,
-0.16170787803919912
],
"resamples": 2000
}
},
"bench_lv_vqav2_yesno": {
"acc": {
"metric": "accuracy",
"questions": 4887,
"pictures": 2069,
"a": 0.7249846531614488,
"b": 0.7558829547779824,
"difference": 0.03089830161653362,
"ci95": [
0.01905156347069792,
0.04241915161696774
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 4887,
"pictures": 2069,
"a": 0.551730515857613,
"b": 0.5256937856914787,
"difference": -0.02603673016613428,
"ci95": [
-0.03484135061656102,
-0.01729670636981273
],
"resamples": 2000
}
},
"bench_lv_aokvqa": {
"acc": {
"metric": "accuracy",
"questions": 1138,
"pictures": 1138,
"a": 0.6124780316344464,
"b": 0.6335676625659051,
"difference": 0.02108963093145877,
"ci95": [
-0.0035149384885764245,
0.0448154657293498
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1138,
"pictures": 1138,
"a": 0.9357384620349536,
"b": 0.9235836608133525,
"difference": -0.012154801221601064,
"ci95": [
-0.050025256755278025,
0.02747038629963383
],
"resamples": 2000
}
},
"bench_lv_scienceqa": {
"acc": {
"metric": "accuracy",
"questions": 2097,
"pictures": 2097,
"a": 0.7892226990939437,
"b": 0.8206962327134001,
"difference": 0.03147353361945637,
"ci95": [
0.015736766809728242,
0.047687172150691515
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2097,
"pictures": 2097,
"a": 0.47098928326742956,
"b": 0.46387570466731803,
"difference": -0.007113578600111525,
"ci95": [
-0.036043017961029544,
0.02201631592471548
],
"resamples": 2000
}
},
"bench_whatsup": {
"acc": {
"metric": "accuracy",
"questions": 820,
"pictures": 820,
"a": 0.2865853658536585,
"b": 0.46707317073170734,
"difference": 0.18048780487804883,
"ci95": [
0.1353658536585366,
0.22073170731707314
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 820,
"pictures": 820,
"a": 1.490915758604817,
"b": 1.2141925113554008,
"difference": -0.2767232472494161,
"ci95": [
-0.335244532597462,
-0.219079539978984
],
"resamples": 2000
}
},
"bench_countbench": {
"acc": {
"metric": "accuracy",
"questions": 491,
"pictures": 491,
"a": 0.3564154786150713,
"b": 0.5010183299389002,
"difference": 0.14460285132382888,
"ci95": [
0.09368635437881867,
0.1955193482688391
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 491,
"pictures": 491,
"a": 1.6021549698788784,
"b": 1.3872406640068242,
"difference": -0.21491430587205418,
"ci95": [
-0.2562705789097375,
-0.17616865330760545
],
"resamples": 2000
}
},
"bench_cogent_b": {
"acc": {
"metric": "accuracy",
"questions": 2000,
"pictures": 2000,
"a": 0.5775,
"b": 0.618,
"difference": 0.04049999999999998,
"ci95": [
0.01649999999999996,
0.0655
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2000,
"pictures": 2000,
"a": 0.8426200623459391,
"b": 0.8368894876234722,
"difference": -0.005730574722466963,
"ci95": [
-0.02600691985620429,
0.013703787042339766
],
"resamples": 2000
}
},
"test_count_coco": {
"acc": {
"metric": "accuracy",
"questions": 1100,
"pictures": 1100,
"a": 0.30636363636363634,
"b": 0.34454545454545454,
"difference": 0.038181818181818206,
"ci95": [
0.011818181818181783,
0.06545454545454549
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1100,
"pictures": 1100,
"a": 1.9893882985025666,
"b": 1.779380167228201,
"difference": -0.21000813127436557,
"ci95": [
-0.25964636612728054,
-0.1622590152983625
],
"resamples": 2000
}
},
"bench_mmbench": {
"acc": {
"metric": "accuracy",
"questions": 4294,
"pictures": 4294,
"a": 0.6040987424312995,
"b": 0.6266884024219842,
"difference": 0.02258965999068474,
"ci95": [
0.011178388448998544,
0.03446669771774569
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 4294,
"pictures": 4294,
"a": 0.9621266293403985,
"b": 0.9076105848749486,
"difference": -0.05451604446544989,
"ci95": [
-0.07615708256293019,
-0.03108140813878635
],
"resamples": 2000
}
},
"bench_cvbench": {
"acc": {
"metric": "accuracy",
"questions": 2638,
"pictures": 2638,
"a": 0.5443517816527672,
"b": 0.5511751326762699,
"difference": 0.006823351023502733,
"ci95": [
-0.009855951478392688,
0.02501895375284313
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2638,
"pictures": 2638,
"a": 0.9016688344057614,
"b": 0.8083227674466051,
"difference": -0.09334606695915626,
"ci95": [
-0.11244997195903339,
-0.07520308890416327
],
"resamples": 2000
}
},
"bench_mme": {
"acc": {
"metric": "accuracy",
"questions": 2374,
"pictures": 1187,
"a": 0.6533277169334457,
"b": 0.6474304970513901,
"difference": -0.005897219882055604,
"ci95": [
-0.02064026958719467,
0.008845829823083351
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2374,
"pictures": 1187,
"a": 0.6057342796352463,
"b": 0.6429894737804899,
"difference": 0.037255194145243586,
"ci95": [
0.023705705678477296,
0.05017788218628971
],
"resamples": 2000
}
},
"bench_naturalbench": {
"acc": {
"metric": "accuracy",
"questions": 7570,
"pictures": 3800,
"a": 0.6206076618229854,
"b": 0.6240422721268164,
"difference": 0.003434610303830965,
"ci95": [
-0.007000396248844298,
0.013199577613516311
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 7570,
"pictures": 3800,
"a": 0.6743440115208639,
"b": 0.6925552586321518,
"difference": 0.018211247111287965,
"ci95": [
0.008188317325104721,
0.029288391972986494
],
"resamples": 2000
}
},
"dev_mix": {
"acc": {
"metric": "accuracy",
"questions": 5178,
"pictures": 3042,
"a": 0.7576284279644651,
"b": 0.790845886442642,
"difference": 0.03321745847817692,
"ci95": [
0.022335794056407288,
0.04369682907965966
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 5178,
"pictures": 3042,
"a": 0.5432908524336165,
"b": 0.5260876670578333,
"difference": -0.017203185375783248,
"ci95": [
-0.034758412199066724,
0.0017373233636809315
],
"resamples": 2000
}
},
"dev_sqa": {
"acc": {
"metric": "accuracy",
"questions": 2017,
"pictures": 1799,
"a": 0.8041646008924145,
"b": 0.8463063956370848,
"difference": 0.04214179474467028,
"ci95": [
0.024938875305623442,
0.05867970660146704
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2017,
"pictures": 1799,
"a": 0.45615675456359894,
"b": 0.416725354374512,
"difference": -0.03943140018908692,
"ci95": [
-0.06878795828969625,
-0.007295041475122277
],
"resamples": 2000
}
}
},
"verdict": {
"missing": [],
"1_no_set_worse": {
"pass": true,
"worse": []
},
"2_at_least_15_up": {
"pass": true,
"up": 22,
"of": 23
},
"3_weak_4_of_5": {
"pass": true,
"by_ability": {
"ScienceQA": true,
"VSR": true,
"counting": true,
"A-OKVQA": false,
"V7W": true
}
},
"4_nll_not_worse": {
"pass": false,
"worse": [
"test_exist",
"test_size",
"bench_mme"
]
}
}
}