{ "sets": { "test_chartqa": { "role": "\u65b0\u7528\u9014", "pass": true, "accuracy": { "metric": "accuracy", "questions": 2215, "pictures": 1434, "a": 0.30699774266365687, "b": 0.5891647855530474, "difference": 0.28216704288939054, "ci95": [ 0.2573496155585708, 0.3092224231464738 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2215, "pictures": 1434, "a": 1.3696650694854122, "b": 0.8779470401626752, "difference": -0.491718029322737, "ci95": [ -0.529836041084054, -0.4563086219502238 ], "resamples": 2000 }, "ece": [ 0.08100577490593457, 0.03411409044911577 ] }, "test_gui_shop_state": { "role": "\u65b0\u7528\u9014", "pass": true, "accuracy": { "metric": "accuracy", "questions": 774, "pictures": 774, "a": 0.2144702842377261, "b": 0.6847545219638242, "difference": 0.4702842377260982, "ci95": [ 0.4276485788113695, 0.5116279069767442 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 774, "pictures": 774, "a": 1.6691573627139369, "b": 0.8158972475196088, "difference": -0.8532601151943281, "ci95": [ -0.9333054191135438, -0.7729055863044242 ], "resamples": 2000 }, "ece": [ 0.1254376165896115, 0.08363713154651094 ] }, "test_gui_state": { "role": "\u65b0\u7528\u9014", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1540, "pictures": 1540, "a": 0.2532467532467532, "b": 0.5240259740259741, "difference": 0.27077922077922084, "ci95": [ 0.24025974025974023, 0.3012987012987013 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1540, "pictures": 1540, "a": 1.636095547398147, "b": 1.2587241447371622, "difference": -0.3773714026609849, "ci95": [ -0.4369500354150804, -0.3148551575986829 ], "resamples": 2000 }, "ece": [ 0.12280357762590631, 0.1496722247112881 ] }, "test_gui": { "role": "\u65b0\u7528\u9014", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1540, "pictures": 1540, "a": 0.38571428571428573, "b": 0.7824675324675324, "difference": 0.3967532467532467, "ci95": [ 0.3681818181818181, 0.4266233766233767 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1540, "pictures": 1540, "a": 1.5296691984309843, "b": 0.6209756534220569, "difference": -0.9086935450089274, "ci95": [ -0.9605080421099128, -0.855393762257445 ], "resamples": 2000 }, "ece": [ 0.04676434546709063, 0.0392522416041269 ] }, "test_ego": { "role": "\u65b0\u7528\u9014", "pass": true, "accuracy": { "metric": "accuracy", "questions": 5864, "pictures": 2957, "a": 0.34703274215552526, "b": 0.7165757162346521, "difference": 0.36954297407912684, "ci95": [ 0.3537656197230666, 0.3867459643160578 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 5864, "pictures": 2957, "a": 1.517406704753598, "b": 0.7081127987428508, "difference": -0.8092939060107471, "ci95": [ -0.8428533030522657, -0.77654791566865 ], "resamples": 2000 }, "ece": [ 0.15014078196819164, 0.055333862041147326 ] }, "bench_pope": { "role": "POPE", "pass": true, "accuracy": { "metric": "accuracy", "questions": 9000, "pictures": 500, "a": 0.8653333333333333, "b": 0.8606666666666667, "difference": -0.004666666666666597, "ci95": [ -0.010666666666666713, 0.0013333333333332975 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 9000, "pictures": 500, "a": 0.36302066313255416, "b": 0.3615067540475159, "difference": -0.0015139090850382342, "ci95": [ -0.009711861690059886, 0.006685591695626492 ], "resamples": 2000 }, "ece": [ 0.048274166814982915, 0.02849606340461299 ] }, "bench_lv_pope": { "role": "POPE", "pass": true, "accuracy": { "metric": "accuracy", "questions": 9000, "pictures": 500, "a": 0.8653333333333333, "b": 0.8606666666666667, "difference": -0.004666666666666597, "ci95": [ -0.010666666666666713, 0.0013333333333332975 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 9000, "pictures": 500, "a": 0.36302066313255416, "b": 0.3615067540475159, "difference": -0.0015139090850382342, "ci95": [ -0.009711861690059886, 0.006685591695626492 ], "resamples": 2000 }, "ece": [ 0.048274166814982915, 0.02849606340461299 ] }, "test_vsr": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 902, "pictures": 547, "a": 0.7305986696230599, "b": 0.7184035476718403, "difference": -0.012195121951219523, "ci95": [ -0.033991228070175405, 0.010250569476081939 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 902, "pictures": 547, "a": 0.5904692457609072, "b": 0.5901306910319974, "difference": -0.0003385547289097568, "ci95": [ -0.02994199540187792, 0.030065255938966917 ], "resamples": 2000 }, "ece": [ 0.073440522495492, 0.10218281392111317 ] }, "test_v7w": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1000, "pictures": 1000, "a": 0.783, "b": 0.777, "difference": -0.006000000000000005, "ci95": [ -0.02200000000000002, 0.009000000000000008 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1000, "pictures": 1000, "a": 0.5966842790244825, "b": 0.5846205238920463, "difference": -0.012063755132436182, "ci95": [ -0.02729742896464138, 0.004461494727335924 ], "resamples": 2000 }, "ece": [ 0.07302743381261825, 0.04071881279349328 ] }, "test_vqa_yesno": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1000, "pictures": 977, "a": 0.722, "b": 0.721, "difference": -0.0010000000000000009, "ci95": [ -0.01501501501501501, 0.015000000000000013 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1000, "pictures": 977, "a": 0.571303429093119, "b": 0.5673028794284216, "difference": -0.004000549664697428, "ci95": [ -0.01661778898958577, 0.008211893208576515 ], "resamples": 2000 }, "ece": [ 0.0551844739280641, 0.062150504142045954 ] }, "test_vqa_choice": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1453, "pictures": 1394, "a": 0.9050240880935995, "b": 0.9050240880935995, "difference": 0.0, "ci95": [ -0.009608785175017176, 0.009675190048376026 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1453, "pictures": 1394, "a": 0.28024055404603077, "b": 0.25107046831169766, "difference": -0.029170085734333107, "ci95": [ -0.0640833216129878, -0.004317750936707443 ], "resamples": 2000 }, "ece": [ 0.05335439617520601, 0.026538925915853925 ] }, "test_gqa": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 986, "pictures": 880, "a": 0.8427991886409736, "b": 0.8448275862068966, "difference": 0.0020283975659229903, "ci95": [ -0.014227642276422814, 0.018924302788844605 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 986, "pictures": 880, "a": 0.38284475493705167, "b": 0.3461517125972417, "difference": -0.03669304233980997, "ci95": [ -0.054851090834021954, -0.018162514381280415 ], "resamples": 2000 }, "ece": [ 0.08570437094019104, 0.02913439515638064 ] }, "test_exist": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1120, "pictures": 1104, "a": 0.9553571428571429, "b": 0.9517857142857142, "difference": -0.0035714285714286698, "ci95": [ -0.011596788581623607, 0.004460303300624413 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1120, "pictures": 1104, "a": 0.19567675111207672, "b": 0.18881174189452032, "difference": -0.006865009217556395, "ci95": [ -0.01630036044872843, 0.0034178696089074423 ], "resamples": 2000 }, "ece": [ 0.09238033705790129, 0.07048237314447756 ] }, "test_position": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1276, "pictures": 1232, "a": 0.9380877742946708, "b": 0.9412225705329154, "difference": 0.0031347962382445305, "ci95": [ -0.007824726134585291, 0.014062499999999978 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1276, "pictures": 1232, "a": 0.23840018591039677, "b": 0.18176862680910122, "difference": -0.05663155910129555, "ci95": [ -0.09586512683435014, -0.016434284545971317 ], "resamples": 2000 }, "ece": [ 0.04677146336875358, 0.040692715108581426 ] }, "test_relation": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 2366, "pictures": 1768, "a": 0.9167371090448013, "b": 0.9120879120879121, "difference": -0.004649196956889257, "ci95": [ -0.013174670633234187, 0.004661016949152552 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2366, "pictures": 1768, "a": 0.2805283450848128, "b": 0.2263127510369931, "difference": -0.05421559404781967, "ci95": [ -0.08689217534583718, -0.025488199506897125 ], "resamples": 2000 }, "ece": [ 0.03667280352194295, 0.028100870714018247 ] }, "test_size": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1198, "pictures": 1081, "a": 0.8697829716193656, "b": 0.8764607679465777, "difference": 0.006677796327212104, "ci95": [ -0.006728343145500415, 0.02010050251256279 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1198, "pictures": 1081, "a": 0.5065374303761236, "b": 0.41590682598351103, "difference": -0.09063060439261261, "ci95": [ -0.09927571801832574, -0.08175392027004708 ], "resamples": 2000 }, "ece": [ 0.23446257026844314, 0.15345264919015122 ] }, "test_position_flip": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 640, "pictures": 636, "a": 0.9328125, "b": 0.940625, "difference": 0.0078125, "ci95": [ -0.006259780907668322, 0.02336448598130836 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 640, "pictures": 636, "a": 0.28297517066909245, "b": 0.1773505214318133, "difference": -0.10562464923727916, "ci95": [ -0.1635211311127471, -0.05430717430676374 ], "resamples": 2000 }, "ece": [ 0.05324600581079723, 0.03369712298735979 ] }, "test_relation_flip": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1786, "pictures": 1460, "a": 0.9143337066069429, "b": 0.9227323628219485, "difference": 0.008398656215005573, "ci95": [ -0.002245929253228529, 0.0196629213483146 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1786, "pictures": 1460, "a": 0.3004006765189471, "b": 0.20978638120140444, "difference": -0.09061429531754264, "ci95": [ -0.13230767349144507, -0.0525172313740668 ], "resamples": 2000 }, "ece": [ 0.048533593783494464, 0.01960618924954432 ] }, "test_count_coco": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1100, "pictures": 1100, "a": 0.34454545454545454, "b": 0.35, "difference": 0.005454545454545434, "ci95": [ -0.014545454545454584, 0.027272727272727282 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1100, "pictures": 1100, "a": 1.779380167228201, "b": 1.7173822043565836, "difference": -0.06199796287161741, "ci95": [ -0.07555289305722424, -0.04827953038811028 ], "resamples": 2000 }, "ece": [ 0.07740107010711325, 0.046413114199584174 ] }, "bench_lv_vqav2_yesno": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 4887, "pictures": 2069, "a": 0.7558829547779824, "b": 0.7493349703294455, "difference": -0.006547984448536881, "ci95": [ -0.014215080346106301, 0.0008324661810613643 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 4887, "pictures": 2069, "a": 0.5256937856914787, "b": 0.5278807692464372, "difference": 0.0021869835549584993, "ci95": [ -0.0027982273082286513, 0.00717907609742785 ], "resamples": 2000 }, "ece": [ 0.03821689732325501, 0.029946236714723785 ] }, "bench_lv_aokvqa": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 1138, "pictures": 1138, "a": 0.6335676625659051, "b": 0.6274165202108963, "difference": -0.0061511423550087985, "ci95": [ -0.024604569420035083, 0.012302284710017597 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 1138, "pictures": 1138, "a": 0.9235836608133525, "b": 0.9195321252696066, "difference": -0.004051535543745932, "ci95": [ -0.030753943518912363, 0.023055812364578876 ], "resamples": 2000 }, "ece": [ 0.04262970552922132, 0.05943972610840687 ] }, "bench_lv_scienceqa": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 2097, "pictures": 2097, "a": 0.8206962327134001, "b": 0.8206962327134001, "difference": 0.0, "ci95": [ -0.012875536480686733, 0.012398664759179723 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2097, "pictures": 2097, "a": 0.46387570466731803, "b": 0.4401898182631203, "difference": -0.02368588640419772, "ci95": [ -0.05354985349065028, 0.00549494597644079 ], "resamples": 2000 }, "ece": [ 0.03733375386419786, 0.0582982054709479 ] }, "bench_whatsup": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 820, "pictures": 820, "a": 0.46707317073170734, "b": 0.5182926829268293, "difference": 0.05121951219512194, "ci95": [ 0.02682926829268295, 0.07560975609756099 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 820, "pictures": 820, "a": 1.2141925113554008, "b": 1.17432266136941, "difference": -0.03986984998599086, "ci95": [ -0.062498376492541086, -0.0180387147623835 ], "resamples": 2000 }, "ece": [ 0.09258908931075073, 0.09595172103585266 ] }, "bench_countbench": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 491, "pictures": 491, "a": 0.5010183299389002, "b": 0.515274949083503, "difference": 0.014256619144602856, "ci95": [ -0.02647657841140527, 0.052953156822810654 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 491, "pictures": 491, "a": 1.3872406640068242, "b": 1.2605759742809557, "difference": -0.12666468972586853, "ci95": [ -0.16063806529157376, -0.09031937883987284 ], "resamples": 2000 }, "ece": [ 0.16553073321789685, 0.07907822530891159 ] }, "bench_cogent_b": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 2000, "pictures": 2000, "a": 0.618, "b": 0.6015, "difference": -0.01649999999999996, "ci95": [ -0.034499999999999975, 0.0010000000000000009 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2000, "pictures": 2000, "a": 0.8368894876234722, "b": 0.8328083746100713, "difference": -0.004081113013400861, "ci95": [ -0.018932167270295674, 0.010907585437141099 ], "resamples": 2000 }, "ece": [ 0.07893497987091541, 0.04171491627395153 ] }, "bench_mmbench": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 4294, "pictures": 4294, "a": 0.6266884024219842, "b": 0.6348393106660456, "difference": 0.008150908244061394, "ci95": [ -0.0016301816488122345, 0.018164881229622676 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 4294, "pictures": 4294, "a": 0.9076105848749486, "b": 0.9029981663095357, "difference": -0.004612418565412946, "ci95": [ -0.021383906693205557, 0.01400348896304815 ], "resamples": 2000 }, "ece": [ 0.07182964694377816, 0.10001497634161224 ] }, "bench_cvbench": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 2638, "pictures": 2638, "a": 0.5511751326762699, "b": 0.6080363912054587, "difference": 0.056861258529188774, "ci95": [ 0.040181956027293464, 0.0727824109173616 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2638, "pictures": 2638, "a": 0.8083227674466051, "b": 0.7711637663698337, "difference": -0.03715900107677139, "ci95": [ -0.047801615800607045, -0.026597798361790215 ], "resamples": 2000 }, "ece": [ 0.03691194208735672, 0.042680238060783124 ] }, "bench_mme": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 2374, "pictures": 1187, "a": 0.6474304970513901, "b": 0.6588037068239259, "difference": 0.011373209772535753, "ci95": [ -0.0008424599831508006, 0.024010109519797762 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2374, "pictures": 1187, "a": 0.6429894737804899, "b": 0.6208586168674086, "difference": -0.022130856913081254, "ci95": [ -0.03162534790460947, -0.012716727642455461 ], "resamples": 2000 }, "ece": [ 0.1105259721270031, 0.08625262097016971 ] }, "bench_naturalbench": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 7570, "pictures": 3800, "a": 0.6240422721268164, "b": 0.6287978863936592, "difference": 0.004755614266842789, "ci95": [ -0.0029058248580108303, 0.011627906976744207 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 7570, "pictures": 3800, "a": 0.6925552586321518, "b": 0.721895445972474, "difference": 0.029340187340322177, "ci95": [ 0.020829243041758705, 0.037686018303601965 ], "resamples": 2000 }, "ece": [ 0.11268248255493778, 0.13449080768244306 ] }, "bench_realworld": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 587, "pictures": 585, "a": 0.46678023850085176, "b": 0.46678023850085176, "difference": 0.0, "ci95": [ -0.030612244897959218, 0.030560271646859094 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 587, "pictures": 585, "a": 1.0100310522774578, "b": 1.0256329281827516, "difference": 0.015601875905293827, "ci95": [ -0.014939422104096156, 0.04679560015709716 ], "resamples": 2000 }, "ece": [ 0.10917688364216985, 0.1384793062023247 ] }, "bench_egothink": { "role": "\u65e7\u80fd\u529b", "pass": true, "accuracy": { "metric": "accuracy", "questions": 604, "pictures": 604, "a": 0.7466887417218543, "b": 0.7748344370860927, "difference": 0.028145695364238388, "ci95": [ -0.0033112582781457123, 0.05794701986754969 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 604, "pictures": 604, "a": 0.6478925168047277, "b": 0.5900769565865127, "difference": -0.05781556021821499, "ci95": [ -0.10609937995123098, -0.010686242502153775 ], "resamples": 2000 }, "ece": [ 0.05624508364311119, 0.03441751555891223 ] }, "dev_mix": { "role": "\u62a5\u544a", "pass": null, "accuracy": { "metric": "accuracy", "questions": 5178, "pictures": 3042, "a": 0.790845886442642, "b": 0.7970258787176516, "difference": 0.006179992275009649, "ci95": [ -0.001548287207276955, 0.013880626611144198 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 5178, "pictures": 3042, "a": 0.5260876670578333, "b": 0.5063605608498457, "difference": -0.019727106207987566, "ci95": [ -0.032697972427902955, -0.0068425494509643525 ], "resamples": 2000 }, "ece": [ 0.03974227216974603, 0.024395532998863947 ] }, "dev_sqa": { "role": "\u62a5\u544a", "pass": null, "accuracy": { "metric": "accuracy", "questions": 2017, "pictures": 1799, "a": 0.8463063956370848, "b": 0.8542389687654933, "difference": 0.00793257312840856, "ci95": [ -0.0050226017076845375, 0.022168674698795243 ], "resamples": 2000 }, "nll": { "metric": "nll", "questions": 2017, "pictures": 1799, "a": 0.416725354374512, "b": 0.3755989106004386, "difference": -0.04112644377407343, "ci95": [ -0.07071182748751992, -0.012094534839404925 ], "resamples": 2000 }, "ece": [ 0.041378608384401896, 0.041724693999765625 ] } }, "rule1": true, "rule2": true, "nll_and_ece_worse": [ "bench_naturalbench", "bench_realworld" ] }