Download evaluation/accept_vs_v0.3.json from lukbit/devision: direct link, hf CLI and curl.
- Browser
- Download file 14.9 kB
-
https://huggingface.co/lukbit/devision/resolve/main/evaluation/accept_vs_v0.3.json
- Command line
-
hf download hf://lukbit/devision/evaluation/accept_vs_v0.3.json
-
curl -L -o accept_vs_v0.3.json https://huggingface.co/lukbit/devision/resolve/main/evaluation/accept_vs_v0.3.json
14.9 kB
| { | |
| "rows": { | |
| "test_vsr": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 902, | |
| "pictures": 547, | |
| "a": 0.6607538802660754, | |
| "b": 0.7305986696230599, | |
| "difference": 0.06984478935698446, | |
| "ci95": [ | |
| 0.03618421052631582, | |
| 0.10647639956092214 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 902, | |
| "pictures": 547, | |
| "a": 0.6155837500135749, | |
| "b": 0.5904692457609072, | |
| "difference": -0.025114504252667702, | |
| "ci95": [ | |
| -0.06730200801082653, | |
| 0.01835022339389425 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_v7w": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 1000, | |
| "pictures": 1000, | |
| "a": 0.711, | |
| "b": 0.783, | |
| "difference": 0.07200000000000006, | |
| "ci95": [ | |
| 0.04800000000000004, | |
| 0.09600000000000009 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 1000, | |
| "pictures": 1000, | |
| "a": 0.7061062017078975, | |
| "b": 0.5966842790244825, | |
| "difference": -0.109421922683415, | |
| "ci95": [ | |
| -0.14233793828593067, | |
| -0.07693998792887025 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_pope": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 9000, | |
| "pictures": 500, | |
| "a": 0.8481111111111111, | |
| "b": 0.8653333333333333, | |
| "difference": 0.01722222222222214, | |
| "ci95": [ | |
| 0.00855555555555565, | |
| 0.02577777777777779 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 9000, | |
| "pictures": 500, | |
| "a": 0.37213111549634703, | |
| "b": 0.36302066313255416, | |
| "difference": -0.009110452363792876, | |
| "ci95": [ | |
| -0.02169886047608327, | |
| 0.003435790059213739 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_vqa_yesno": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 1000, | |
| "pictures": 977, | |
| "a": 0.683, | |
| "b": 0.722, | |
| "difference": 0.038999999999999924, | |
| "ci95": [ | |
| 0.015090543259557276, | |
| 0.06480558325024932 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 1000, | |
| "pictures": 977, | |
| "a": 0.5941348191891218, | |
| "b": 0.571303429093119, | |
| "difference": -0.022831390096002768, | |
| "ci95": [ | |
| -0.04409385128971266, | |
| -0.0019899478295204442 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_vqa_choice": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 1453, | |
| "pictures": 1394, | |
| "a": 0.8823124569855472, | |
| "b": 0.9050240880935995, | |
| "difference": 0.0227116311080523, | |
| "ci95": [ | |
| 0.007686932215234088, | |
| 0.03782668500687758 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 1453, | |
| "pictures": 1394, | |
| "a": 0.2975529214462992, | |
| "b": 0.28024055404603077, | |
| "difference": -0.017312367400268425, | |
| "ci95": [ | |
| -0.04801307597139293, | |
| 0.017763019358089716 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_gqa": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 986, | |
| "pictures": 880, | |
| "a": 0.781947261663286, | |
| "b": 0.8427991886409736, | |
| "difference": 0.0608519269776876, | |
| "ci95": [ | |
| 0.03567787971457692, | |
| 0.08611955420466055 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 986, | |
| "pictures": 880, | |
| "a": 0.4363967446492391, | |
| "b": 0.38284475493705167, | |
| "difference": -0.05355198971218744, | |
| "ci95": [ | |
| -0.08716986207647759, | |
| -0.02041387915707188 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_exist": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 1120, | |
| "pictures": 1104, | |
| "a": 0.9392857142857143, | |
| "b": 0.9553571428571429, | |
| "difference": 0.016071428571428625, | |
| "ci95": [ | |
| 0.0026929982046678402, | |
| 0.02930728241563063 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 1120, | |
| "pictures": 1104, | |
| "a": 0.16983095379080976, | |
| "b": 0.19567675111207672, | |
| "difference": 0.02584579732126696, | |
| "ci95": [ | |
| 0.007774706565661066, | |
| 0.04337850870878224 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_position": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 1276, | |
| "pictures": 1232, | |
| "a": 0.8863636363636364, | |
| "b": 0.9380877742946708, | |
| "difference": 0.051724137931034475, | |
| "ci95": [ | |
| 0.03299293008641002, | |
| 0.07092198581560283 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 1276, | |
| "pictures": 1232, | |
| "a": 0.27279601591009106, | |
| "b": 0.23840018591039677, | |
| "difference": -0.03439582999969429, | |
| "ci95": [ | |
| -0.08219989842410494, | |
| 0.015482905705434202 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_relation": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 2366, | |
| "pictures": 1768, | |
| "a": 0.7489433643279797, | |
| "b": 0.9167371090448013, | |
| "difference": 0.16779374471682162, | |
| "ci95": [ | |
| 0.1491782553729456, | |
| 0.18608549874266556 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 2366, | |
| "pictures": 1768, | |
| "a": 0.4909602253543323, | |
| "b": 0.2805283450848128, | |
| "difference": -0.21043188026951953, | |
| "ci95": [ | |
| -0.25335914427956835, | |
| -0.16455708571446082 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_size": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 1198, | |
| "pictures": 1081, | |
| "a": 0.8564273789649416, | |
| "b": 0.8697829716193656, | |
| "difference": 0.013355592654423987, | |
| "ci95": [ | |
| -0.002523128679562725, | |
| 0.030252100840336138 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 1198, | |
| "pictures": 1081, | |
| "a": 0.35047560484582363, | |
| "b": 0.5065374303761236, | |
| "difference": 0.1560618255303, | |
| "ci95": [ | |
| 0.12203261708552204, | |
| 0.18841077280205148 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_position_flip": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 640, | |
| "pictures": 636, | |
| "a": 0.828125, | |
| "b": 0.9328125, | |
| "difference": 0.10468750000000004, | |
| "ci95": [ | |
| 0.07931570762052875, | |
| 0.1316614420062696 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 640, | |
| "pictures": 636, | |
| "a": 0.3792944504698811, | |
| "b": 0.28297517066909245, | |
| "difference": -0.09631927980078864, | |
| "ci95": [ | |
| -0.16729839354099188, | |
| -0.019257792204828483 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_relation_flip": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 1786, | |
| "pictures": 1460, | |
| "a": 0.7021276595744681, | |
| "b": 0.9143337066069429, | |
| "difference": 0.2122060470324748, | |
| "ci95": [ | |
| 0.18990246701090074, | |
| 0.2346089850249583 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 1786, | |
| "pictures": 1460, | |
| "a": 0.5234402751432231, | |
| "b": 0.3004006765189471, | |
| "difference": -0.22303959862427603, | |
| "ci95": [ | |
| -0.27563722667679236, | |
| -0.16170787803919912 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_lv_vqav2_yesno": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 4887, | |
| "pictures": 2069, | |
| "a": 0.7249846531614488, | |
| "b": 0.7558829547779824, | |
| "difference": 0.03089830161653362, | |
| "ci95": [ | |
| 0.01905156347069792, | |
| 0.04241915161696774 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 4887, | |
| "pictures": 2069, | |
| "a": 0.551730515857613, | |
| "b": 0.5256937856914787, | |
| "difference": -0.02603673016613428, | |
| "ci95": [ | |
| -0.03484135061656102, | |
| -0.01729670636981273 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_lv_aokvqa": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 1138, | |
| "pictures": 1138, | |
| "a": 0.6124780316344464, | |
| "b": 0.6335676625659051, | |
| "difference": 0.02108963093145877, | |
| "ci95": [ | |
| -0.0035149384885764245, | |
| 0.0448154657293498 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 1138, | |
| "pictures": 1138, | |
| "a": 0.9357384620349536, | |
| "b": 0.9235836608133525, | |
| "difference": -0.012154801221601064, | |
| "ci95": [ | |
| -0.050025256755278025, | |
| 0.02747038629963383 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_lv_scienceqa": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 2097, | |
| "pictures": 2097, | |
| "a": 0.7892226990939437, | |
| "b": 0.8206962327134001, | |
| "difference": 0.03147353361945637, | |
| "ci95": [ | |
| 0.015736766809728242, | |
| 0.047687172150691515 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 2097, | |
| "pictures": 2097, | |
| "a": 0.47098928326742956, | |
| "b": 0.46387570466731803, | |
| "difference": -0.007113578600111525, | |
| "ci95": [ | |
| -0.036043017961029544, | |
| 0.02201631592471548 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_whatsup": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 820, | |
| "pictures": 820, | |
| "a": 0.2865853658536585, | |
| "b": 0.46707317073170734, | |
| "difference": 0.18048780487804883, | |
| "ci95": [ | |
| 0.1353658536585366, | |
| 0.22073170731707314 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 820, | |
| "pictures": 820, | |
| "a": 1.490915758604817, | |
| "b": 1.2141925113554008, | |
| "difference": -0.2767232472494161, | |
| "ci95": [ | |
| -0.335244532597462, | |
| -0.219079539978984 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_countbench": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 491, | |
| "pictures": 491, | |
| "a": 0.3564154786150713, | |
| "b": 0.5010183299389002, | |
| "difference": 0.14460285132382888, | |
| "ci95": [ | |
| 0.09368635437881867, | |
| 0.1955193482688391 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 491, | |
| "pictures": 491, | |
| "a": 1.6021549698788784, | |
| "b": 1.3872406640068242, | |
| "difference": -0.21491430587205418, | |
| "ci95": [ | |
| -0.2562705789097375, | |
| -0.17616865330760545 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_cogent_b": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 2000, | |
| "pictures": 2000, | |
| "a": 0.5775, | |
| "b": 0.618, | |
| "difference": 0.04049999999999998, | |
| "ci95": [ | |
| 0.01649999999999996, | |
| 0.0655 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 2000, | |
| "pictures": 2000, | |
| "a": 0.8426200623459391, | |
| "b": 0.8368894876234722, | |
| "difference": -0.005730574722466963, | |
| "ci95": [ | |
| -0.02600691985620429, | |
| 0.013703787042339766 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "test_count_coco": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 1100, | |
| "pictures": 1100, | |
| "a": 0.30636363636363634, | |
| "b": 0.34454545454545454, | |
| "difference": 0.038181818181818206, | |
| "ci95": [ | |
| 0.011818181818181783, | |
| 0.06545454545454549 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 1100, | |
| "pictures": 1100, | |
| "a": 1.9893882985025666, | |
| "b": 1.779380167228201, | |
| "difference": -0.21000813127436557, | |
| "ci95": [ | |
| -0.25964636612728054, | |
| -0.1622590152983625 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_mmbench": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 4294, | |
| "pictures": 4294, | |
| "a": 0.6040987424312995, | |
| "b": 0.6266884024219842, | |
| "difference": 0.02258965999068474, | |
| "ci95": [ | |
| 0.011178388448998544, | |
| 0.03446669771774569 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 4294, | |
| "pictures": 4294, | |
| "a": 0.9621266293403985, | |
| "b": 0.9076105848749486, | |
| "difference": -0.05451604446544989, | |
| "ci95": [ | |
| -0.07615708256293019, | |
| -0.03108140813878635 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_cvbench": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 2638, | |
| "pictures": 2638, | |
| "a": 0.5443517816527672, | |
| "b": 0.5511751326762699, | |
| "difference": 0.006823351023502733, | |
| "ci95": [ | |
| -0.009855951478392688, | |
| 0.02501895375284313 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 2638, | |
| "pictures": 2638, | |
| "a": 0.9016688344057614, | |
| "b": 0.8083227674466051, | |
| "difference": -0.09334606695915626, | |
| "ci95": [ | |
| -0.11244997195903339, | |
| -0.07520308890416327 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_mme": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 2374, | |
| "pictures": 1187, | |
| "a": 0.6533277169334457, | |
| "b": 0.6474304970513901, | |
| "difference": -0.005897219882055604, | |
| "ci95": [ | |
| -0.02064026958719467, | |
| 0.008845829823083351 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 2374, | |
| "pictures": 1187, | |
| "a": 0.6057342796352463, | |
| "b": 0.6429894737804899, | |
| "difference": 0.037255194145243586, | |
| "ci95": [ | |
| 0.023705705678477296, | |
| 0.05017788218628971 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "bench_naturalbench": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 7570, | |
| "pictures": 3800, | |
| "a": 0.6206076618229854, | |
| "b": 0.6240422721268164, | |
| "difference": 0.003434610303830965, | |
| "ci95": [ | |
| -0.007000396248844298, | |
| 0.013199577613516311 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 7570, | |
| "pictures": 3800, | |
| "a": 0.6743440115208639, | |
| "b": 0.6925552586321518, | |
| "difference": 0.018211247111287965, | |
| "ci95": [ | |
| 0.008188317325104721, | |
| 0.029288391972986494 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "dev_mix": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 5178, | |
| "pictures": 3042, | |
| "a": 0.7576284279644651, | |
| "b": 0.790845886442642, | |
| "difference": 0.03321745847817692, | |
| "ci95": [ | |
| 0.022335794056407288, | |
| 0.04369682907965966 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 5178, | |
| "pictures": 3042, | |
| "a": 0.5432908524336165, | |
| "b": 0.5260876670578333, | |
| "difference": -0.017203185375783248, | |
| "ci95": [ | |
| -0.034758412199066724, | |
| 0.0017373233636809315 | |
| ], | |
| "resamples": 2000 | |
| } | |
| }, | |
| "dev_sqa": { | |
| "acc": { | |
| "metric": "accuracy", | |
| "questions": 2017, | |
| "pictures": 1799, | |
| "a": 0.8041646008924145, | |
| "b": 0.8463063956370848, | |
| "difference": 0.04214179474467028, | |
| "ci95": [ | |
| 0.024938875305623442, | |
| 0.05867970660146704 | |
| ], | |
| "resamples": 2000 | |
| }, | |
| "nll": { | |
| "metric": "nll", | |
| "questions": 2017, | |
| "pictures": 1799, | |
| "a": 0.45615675456359894, | |
| "b": 0.416725354374512, | |
| "difference": -0.03943140018908692, | |
| "ci95": [ | |
| -0.06878795828969625, | |
| -0.007295041475122277 | |
| ], | |
| "resamples": 2000 | |
| } | |
| } | |
| }, | |
| "verdict": { | |
| "missing": [], | |
| "1_no_set_worse": { | |
| "pass": true, | |
| "worse": [] | |
| }, | |
| "2_at_least_15_up": { | |
| "pass": true, | |
| "up": 22, | |
| "of": 23 | |
| }, | |
| "3_weak_4_of_5": { | |
| "pass": true, | |
| "by_ability": { | |
| "ScienceQA": true, | |
| "VSR": true, | |
| "counting": true, | |
| "A-OKVQA": false, | |
| "V7W": true | |
| } | |
| }, | |
| "4_nll_not_worse": { | |
| "pass": false, | |
| "worse": [ | |
| "test_exist", | |
| "test_size", | |
| "bench_mme" | |
| ] | |
| } | |
| } | |
| } |