Download evaluation/accept_vs_v0.4.json from lukbit/devision: direct link, hf CLI and curl.
- Browser
- Download file 23.3 kB
-
https://huggingface.co/lukbit/devision/resolve/main/evaluation/accept_vs_v0.4.json
- Command line
-
hf download hf://lukbit/devision/evaluation/accept_vs_v0.4.json
-
curl -L -o accept_vs_v0.4.json https://huggingface.co/lukbit/devision/resolve/main/evaluation/accept_vs_v0.4.json
23.3 kB
{
"sets": {
"test_chartqa": {
"role": "\u65b0\u7528\u9014",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 2215,
"pictures": 1434,
"a": 0.30699774266365687,
"b": 0.5891647855530474,
"difference": 0.28216704288939054,
"ci95": [
0.2573496155585708,
0.3092224231464738
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2215,
"pictures": 1434,
"a": 1.3696650694854122,
"b": 0.8779470401626752,
"difference": -0.491718029322737,
"ci95": [
-0.529836041084054,
-0.4563086219502238
],
"resamples": 2000
},
"ece": [
0.08100577490593457,
0.03411409044911577
]
},
"test_gui_shop_state": {
"role": "\u65b0\u7528\u9014",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 774,
"pictures": 774,
"a": 0.2144702842377261,
"b": 0.6847545219638242,
"difference": 0.4702842377260982,
"ci95": [
0.4276485788113695,
0.5116279069767442
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 774,
"pictures": 774,
"a": 1.6691573627139369,
"b": 0.8158972475196088,
"difference": -0.8532601151943281,
"ci95": [
-0.9333054191135438,
-0.7729055863044242
],
"resamples": 2000
},
"ece": [
0.1254376165896115,
0.08363713154651094
]
},
"test_gui_state": {
"role": "\u65b0\u7528\u9014",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1540,
"pictures": 1540,
"a": 0.2532467532467532,
"b": 0.5240259740259741,
"difference": 0.27077922077922084,
"ci95": [
0.24025974025974023,
0.3012987012987013
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1540,
"pictures": 1540,
"a": 1.636095547398147,
"b": 1.2587241447371622,
"difference": -0.3773714026609849,
"ci95": [
-0.4369500354150804,
-0.3148551575986829
],
"resamples": 2000
},
"ece": [
0.12280357762590631,
0.1496722247112881
]
},
"test_gui": {
"role": "\u65b0\u7528\u9014",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1540,
"pictures": 1540,
"a": 0.38571428571428573,
"b": 0.7824675324675324,
"difference": 0.3967532467532467,
"ci95": [
0.3681818181818181,
0.4266233766233767
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1540,
"pictures": 1540,
"a": 1.5296691984309843,
"b": 0.6209756534220569,
"difference": -0.9086935450089274,
"ci95": [
-0.9605080421099128,
-0.855393762257445
],
"resamples": 2000
},
"ece": [
0.04676434546709063,
0.0392522416041269
]
},
"test_ego": {
"role": "\u65b0\u7528\u9014",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 5864,
"pictures": 2957,
"a": 0.34703274215552526,
"b": 0.7165757162346521,
"difference": 0.36954297407912684,
"ci95": [
0.3537656197230666,
0.3867459643160578
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 5864,
"pictures": 2957,
"a": 1.517406704753598,
"b": 0.7081127987428508,
"difference": -0.8092939060107471,
"ci95": [
-0.8428533030522657,
-0.77654791566865
],
"resamples": 2000
},
"ece": [
0.15014078196819164,
0.055333862041147326
]
},
"bench_pope": {
"role": "POPE",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 9000,
"pictures": 500,
"a": 0.8653333333333333,
"b": 0.8606666666666667,
"difference": -0.004666666666666597,
"ci95": [
-0.010666666666666713,
0.0013333333333332975
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 9000,
"pictures": 500,
"a": 0.36302066313255416,
"b": 0.3615067540475159,
"difference": -0.0015139090850382342,
"ci95": [
-0.009711861690059886,
0.006685591695626492
],
"resamples": 2000
},
"ece": [
0.048274166814982915,
0.02849606340461299
]
},
"bench_lv_pope": {
"role": "POPE",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 9000,
"pictures": 500,
"a": 0.8653333333333333,
"b": 0.8606666666666667,
"difference": -0.004666666666666597,
"ci95": [
-0.010666666666666713,
0.0013333333333332975
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 9000,
"pictures": 500,
"a": 0.36302066313255416,
"b": 0.3615067540475159,
"difference": -0.0015139090850382342,
"ci95": [
-0.009711861690059886,
0.006685591695626492
],
"resamples": 2000
},
"ece": [
0.048274166814982915,
0.02849606340461299
]
},
"test_vsr": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 902,
"pictures": 547,
"a": 0.7305986696230599,
"b": 0.7184035476718403,
"difference": -0.012195121951219523,
"ci95": [
-0.033991228070175405,
0.010250569476081939
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 902,
"pictures": 547,
"a": 0.5904692457609072,
"b": 0.5901306910319974,
"difference": -0.0003385547289097568,
"ci95": [
-0.02994199540187792,
0.030065255938966917
],
"resamples": 2000
},
"ece": [
0.073440522495492,
0.10218281392111317
]
},
"test_v7w": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1000,
"pictures": 1000,
"a": 0.783,
"b": 0.777,
"difference": -0.006000000000000005,
"ci95": [
-0.02200000000000002,
0.009000000000000008
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1000,
"pictures": 1000,
"a": 0.5966842790244825,
"b": 0.5846205238920463,
"difference": -0.012063755132436182,
"ci95": [
-0.02729742896464138,
0.004461494727335924
],
"resamples": 2000
},
"ece": [
0.07302743381261825,
0.04071881279349328
]
},
"test_vqa_yesno": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1000,
"pictures": 977,
"a": 0.722,
"b": 0.721,
"difference": -0.0010000000000000009,
"ci95": [
-0.01501501501501501,
0.015000000000000013
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1000,
"pictures": 977,
"a": 0.571303429093119,
"b": 0.5673028794284216,
"difference": -0.004000549664697428,
"ci95": [
-0.01661778898958577,
0.008211893208576515
],
"resamples": 2000
},
"ece": [
0.0551844739280641,
0.062150504142045954
]
},
"test_vqa_choice": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1453,
"pictures": 1394,
"a": 0.9050240880935995,
"b": 0.9050240880935995,
"difference": 0.0,
"ci95": [
-0.009608785175017176,
0.009675190048376026
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1453,
"pictures": 1394,
"a": 0.28024055404603077,
"b": 0.25107046831169766,
"difference": -0.029170085734333107,
"ci95": [
-0.0640833216129878,
-0.004317750936707443
],
"resamples": 2000
},
"ece": [
0.05335439617520601,
0.026538925915853925
]
},
"test_gqa": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 986,
"pictures": 880,
"a": 0.8427991886409736,
"b": 0.8448275862068966,
"difference": 0.0020283975659229903,
"ci95": [
-0.014227642276422814,
0.018924302788844605
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 986,
"pictures": 880,
"a": 0.38284475493705167,
"b": 0.3461517125972417,
"difference": -0.03669304233980997,
"ci95": [
-0.054851090834021954,
-0.018162514381280415
],
"resamples": 2000
},
"ece": [
0.08570437094019104,
0.02913439515638064
]
},
"test_exist": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1120,
"pictures": 1104,
"a": 0.9553571428571429,
"b": 0.9517857142857142,
"difference": -0.0035714285714286698,
"ci95": [
-0.011596788581623607,
0.004460303300624413
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1120,
"pictures": 1104,
"a": 0.19567675111207672,
"b": 0.18881174189452032,
"difference": -0.006865009217556395,
"ci95": [
-0.01630036044872843,
0.0034178696089074423
],
"resamples": 2000
},
"ece": [
0.09238033705790129,
0.07048237314447756
]
},
"test_position": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1276,
"pictures": 1232,
"a": 0.9380877742946708,
"b": 0.9412225705329154,
"difference": 0.0031347962382445305,
"ci95": [
-0.007824726134585291,
0.014062499999999978
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1276,
"pictures": 1232,
"a": 0.23840018591039677,
"b": 0.18176862680910122,
"difference": -0.05663155910129555,
"ci95": [
-0.09586512683435014,
-0.016434284545971317
],
"resamples": 2000
},
"ece": [
0.04677146336875358,
0.040692715108581426
]
},
"test_relation": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 2366,
"pictures": 1768,
"a": 0.9167371090448013,
"b": 0.9120879120879121,
"difference": -0.004649196956889257,
"ci95": [
-0.013174670633234187,
0.004661016949152552
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2366,
"pictures": 1768,
"a": 0.2805283450848128,
"b": 0.2263127510369931,
"difference": -0.05421559404781967,
"ci95": [
-0.08689217534583718,
-0.025488199506897125
],
"resamples": 2000
},
"ece": [
0.03667280352194295,
0.028100870714018247
]
},
"test_size": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1198,
"pictures": 1081,
"a": 0.8697829716193656,
"b": 0.8764607679465777,
"difference": 0.006677796327212104,
"ci95": [
-0.006728343145500415,
0.02010050251256279
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1198,
"pictures": 1081,
"a": 0.5065374303761236,
"b": 0.41590682598351103,
"difference": -0.09063060439261261,
"ci95": [
-0.09927571801832574,
-0.08175392027004708
],
"resamples": 2000
},
"ece": [
0.23446257026844314,
0.15345264919015122
]
},
"test_position_flip": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 640,
"pictures": 636,
"a": 0.9328125,
"b": 0.940625,
"difference": 0.0078125,
"ci95": [
-0.006259780907668322,
0.02336448598130836
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 640,
"pictures": 636,
"a": 0.28297517066909245,
"b": 0.1773505214318133,
"difference": -0.10562464923727916,
"ci95": [
-0.1635211311127471,
-0.05430717430676374
],
"resamples": 2000
},
"ece": [
0.05324600581079723,
0.03369712298735979
]
},
"test_relation_flip": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1786,
"pictures": 1460,
"a": 0.9143337066069429,
"b": 0.9227323628219485,
"difference": 0.008398656215005573,
"ci95": [
-0.002245929253228529,
0.0196629213483146
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1786,
"pictures": 1460,
"a": 0.3004006765189471,
"b": 0.20978638120140444,
"difference": -0.09061429531754264,
"ci95": [
-0.13230767349144507,
-0.0525172313740668
],
"resamples": 2000
},
"ece": [
0.048533593783494464,
0.01960618924954432
]
},
"test_count_coco": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1100,
"pictures": 1100,
"a": 0.34454545454545454,
"b": 0.35,
"difference": 0.005454545454545434,
"ci95": [
-0.014545454545454584,
0.027272727272727282
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1100,
"pictures": 1100,
"a": 1.779380167228201,
"b": 1.7173822043565836,
"difference": -0.06199796287161741,
"ci95": [
-0.07555289305722424,
-0.04827953038811028
],
"resamples": 2000
},
"ece": [
0.07740107010711325,
0.046413114199584174
]
},
"bench_lv_vqav2_yesno": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 4887,
"pictures": 2069,
"a": 0.7558829547779824,
"b": 0.7493349703294455,
"difference": -0.006547984448536881,
"ci95": [
-0.014215080346106301,
0.0008324661810613643
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 4887,
"pictures": 2069,
"a": 0.5256937856914787,
"b": 0.5278807692464372,
"difference": 0.0021869835549584993,
"ci95": [
-0.0027982273082286513,
0.00717907609742785
],
"resamples": 2000
},
"ece": [
0.03821689732325501,
0.029946236714723785
]
},
"bench_lv_aokvqa": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1138,
"pictures": 1138,
"a": 0.6335676625659051,
"b": 0.6274165202108963,
"difference": -0.0061511423550087985,
"ci95": [
-0.024604569420035083,
0.012302284710017597
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1138,
"pictures": 1138,
"a": 0.9235836608133525,
"b": 0.9195321252696066,
"difference": -0.004051535543745932,
"ci95": [
-0.030753943518912363,
0.023055812364578876
],
"resamples": 2000
},
"ece": [
0.04262970552922132,
0.05943972610840687
]
},
"bench_lv_scienceqa": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 2097,
"pictures": 2097,
"a": 0.8206962327134001,
"b": 0.8206962327134001,
"difference": 0.0,
"ci95": [
-0.012875536480686733,
0.012398664759179723
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2097,
"pictures": 2097,
"a": 0.46387570466731803,
"b": 0.4401898182631203,
"difference": -0.02368588640419772,
"ci95": [
-0.05354985349065028,
0.00549494597644079
],
"resamples": 2000
},
"ece": [
0.03733375386419786,
0.0582982054709479
]
},
"bench_whatsup": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 820,
"pictures": 820,
"a": 0.46707317073170734,
"b": 0.5182926829268293,
"difference": 0.05121951219512194,
"ci95": [
0.02682926829268295,
0.07560975609756099
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 820,
"pictures": 820,
"a": 1.2141925113554008,
"b": 1.17432266136941,
"difference": -0.03986984998599086,
"ci95": [
-0.062498376492541086,
-0.0180387147623835
],
"resamples": 2000
},
"ece": [
0.09258908931075073,
0.09595172103585266
]
},
"bench_countbench": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 491,
"pictures": 491,
"a": 0.5010183299389002,
"b": 0.515274949083503,
"difference": 0.014256619144602856,
"ci95": [
-0.02647657841140527,
0.052953156822810654
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 491,
"pictures": 491,
"a": 1.3872406640068242,
"b": 1.2605759742809557,
"difference": -0.12666468972586853,
"ci95": [
-0.16063806529157376,
-0.09031937883987284
],
"resamples": 2000
},
"ece": [
0.16553073321789685,
0.07907822530891159
]
},
"bench_cogent_b": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 2000,
"pictures": 2000,
"a": 0.618,
"b": 0.6015,
"difference": -0.01649999999999996,
"ci95": [
-0.034499999999999975,
0.0010000000000000009
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2000,
"pictures": 2000,
"a": 0.8368894876234722,
"b": 0.8328083746100713,
"difference": -0.004081113013400861,
"ci95": [
-0.018932167270295674,
0.010907585437141099
],
"resamples": 2000
},
"ece": [
0.07893497987091541,
0.04171491627395153
]
},
"bench_mmbench": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 4294,
"pictures": 4294,
"a": 0.6266884024219842,
"b": 0.6348393106660456,
"difference": 0.008150908244061394,
"ci95": [
-0.0016301816488122345,
0.018164881229622676
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 4294,
"pictures": 4294,
"a": 0.9076105848749486,
"b": 0.9029981663095357,
"difference": -0.004612418565412946,
"ci95": [
-0.021383906693205557,
0.01400348896304815
],
"resamples": 2000
},
"ece": [
0.07182964694377816,
0.10001497634161224
]
},
"bench_cvbench": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 2638,
"pictures": 2638,
"a": 0.5511751326762699,
"b": 0.6080363912054587,
"difference": 0.056861258529188774,
"ci95": [
0.040181956027293464,
0.0727824109173616
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2638,
"pictures": 2638,
"a": 0.8083227674466051,
"b": 0.7711637663698337,
"difference": -0.03715900107677139,
"ci95": [
-0.047801615800607045,
-0.026597798361790215
],
"resamples": 2000
},
"ece": [
0.03691194208735672,
0.042680238060783124
]
},
"bench_mme": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 2374,
"pictures": 1187,
"a": 0.6474304970513901,
"b": 0.6588037068239259,
"difference": 0.011373209772535753,
"ci95": [
-0.0008424599831508006,
0.024010109519797762
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2374,
"pictures": 1187,
"a": 0.6429894737804899,
"b": 0.6208586168674086,
"difference": -0.022130856913081254,
"ci95": [
-0.03162534790460947,
-0.012716727642455461
],
"resamples": 2000
},
"ece": [
0.1105259721270031,
0.08625262097016971
]
},
"bench_naturalbench": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 7570,
"pictures": 3800,
"a": 0.6240422721268164,
"b": 0.6287978863936592,
"difference": 0.004755614266842789,
"ci95": [
-0.0029058248580108303,
0.011627906976744207
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 7570,
"pictures": 3800,
"a": 0.6925552586321518,
"b": 0.721895445972474,
"difference": 0.029340187340322177,
"ci95": [
0.020829243041758705,
0.037686018303601965
],
"resamples": 2000
},
"ece": [
0.11268248255493778,
0.13449080768244306
]
},
"bench_realworld": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 587,
"pictures": 585,
"a": 0.46678023850085176,
"b": 0.46678023850085176,
"difference": 0.0,
"ci95": [
-0.030612244897959218,
0.030560271646859094
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 587,
"pictures": 585,
"a": 1.0100310522774578,
"b": 1.0256329281827516,
"difference": 0.015601875905293827,
"ci95": [
-0.014939422104096156,
0.04679560015709716
],
"resamples": 2000
},
"ece": [
0.10917688364216985,
0.1384793062023247
]
},
"bench_egothink": {
"role": "\u65e7\u80fd\u529b",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 604,
"pictures": 604,
"a": 0.7466887417218543,
"b": 0.7748344370860927,
"difference": 0.028145695364238388,
"ci95": [
-0.0033112582781457123,
0.05794701986754969
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 604,
"pictures": 604,
"a": 0.6478925168047277,
"b": 0.5900769565865127,
"difference": -0.05781556021821499,
"ci95": [
-0.10609937995123098,
-0.010686242502153775
],
"resamples": 2000
},
"ece": [
0.05624508364311119,
0.03441751555891223
]
},
"dev_mix": {
"role": "\u62a5\u544a",
"pass": null,
"accuracy": {
"metric": "accuracy",
"questions": 5178,
"pictures": 3042,
"a": 0.790845886442642,
"b": 0.7970258787176516,
"difference": 0.006179992275009649,
"ci95": [
-0.001548287207276955,
0.013880626611144198
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 5178,
"pictures": 3042,
"a": 0.5260876670578333,
"b": 0.5063605608498457,
"difference": -0.019727106207987566,
"ci95": [
-0.032697972427902955,
-0.0068425494509643525
],
"resamples": 2000
},
"ece": [
0.03974227216974603,
0.024395532998863947
]
},
"dev_sqa": {
"role": "\u62a5\u544a",
"pass": null,
"accuracy": {
"metric": "accuracy",
"questions": 2017,
"pictures": 1799,
"a": 0.8463063956370848,
"b": 0.8542389687654933,
"difference": 0.00793257312840856,
"ci95": [
-0.0050226017076845375,
0.022168674698795243
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2017,
"pictures": 1799,
"a": 0.416725354374512,
"b": 0.3755989106004386,
"difference": -0.04112644377407343,
"ci95": [
-0.07071182748751992,
-0.012094534839404925
],
"resamples": 2000
},
"ece": [
0.041378608384401896,
0.041724693999765625
]
}
},
"rule1": true,
"rule2": true,
"nll_and_ece_worse": [
"bench_naturalbench",
"bench_realworld"
]
}
{
"sets": {
"test_chartqa": {
"role": "\u65b0\u7528\u9014",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 2215,
"pictures": 1434,
"a": 0.30699774266365687,
"b": 0.5891647855530474,
"difference": 0.28216704288939054,
"ci95": [
0.2573496155585708,
0.3092224231464738
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 2215,
"pictures": 1434,
"a": 1.3696650694854122,
"b": 0.8779470401626752,
"difference": -0.491718029322737,
"ci95": [
-0.529836041084054,
-0.4563086219502238
],
"resamples": 2000
},
"ece": [
0.08100577490593457,
0.03411409044911577
]
},
"test_gui_shop_state": {
"role": "\u65b0\u7528\u9014",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 774,
"pictures": 774,
"a": 0.2144702842377261,
"b": 0.6847545219638242,
"difference": 0.4702842377260982,
"ci95": [
0.4276485788113695,
0.5116279069767442
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 774,
"pictures": 774,
"a": 1.6691573627139369,
"b": 0.8158972475196088,
"difference": -0.8532601151943281,
"ci95": [
-0.9333054191135438,
-0.7729055863044242
],
"resamples": 2000
},
"ece": [
0.1254376165896115,
0.08363713154651094
]
},
"test_gui_state": {
"role": "\u65b0\u7528\u9014",
"pass": true,
"accuracy": {
"metric": "accuracy",
"questions": 1540,
"pictures": 1540,
"a": 0.2532467532467532,
"b": 0.5240259740259741,
"difference": 0.27077922077922084,
"ci95": [
0.24025974025974023,
0.3012987012987013
],
"resamples": 2000
},
"nll": {
"metric": "nll",
"questions": 1540,
"pictures": 1540,
"a": 1.636095547398147,
"b": 1.2587241447371622,
"difference": -0.3773714026609849,
"ci95": [
-0.4369500354150804,
-0.3148551575986829
],
"resamples": 2000
},
"ece": [