| import json | |
| import re | |
| import string | |
| import warnings | |
| import pandas as pd | |
| import numpy as np | |
| import os | |
| def instruction_scorer(data, judgment_file, model_name): | |
| df = data | |
| img_dict = {} | |
| for j in range(len(df)): | |
| row = df.iloc[j] | |
| img_dict[row['image_url']] = {'category': row['category']} | |
| with open(judgment_file, 'r') as f: | |
| judgements = json.load(f) | |
| model_data = judgements[model_name] | |
| model_analysis = {} | |
| cat = {'time': [0,0], 'shopping': [0,0], 'navigation-transportation': [0,0], 'abstract': [0,0], 'app': [0,0], 'web': [0,0], 'infographics': [0,0], 'stvqa': [0,0], 'estvqa': [0,0]} | |
| count, total = 0, 0 | |
| for key in model_data: | |
| if key in img_dict: | |
| img_data = img_dict[key] | |
| rating = model_data[key] | |
| count += rating | |
| total += 1 | |
| cat[img_data['category']][1] += 1 | |
| cat[img_data['category']][0] += rating | |
| model_analysis[model_name] = {'category': cat} | |
| x = model_analysis[model_name]['category'] | |
| output_dict = {} | |
| for h in x: | |
| output_dict[h]=100*x[h][0]/x[h][1] | |
| output_dict["misc"]= 100 * (x['stvqa'][0] + x['estvqa'][0])/(x['stvqa'][1] + x['stvqa'][1]) | |
| output_dict["average"] = (output_dict["time"]+output_dict["shopping"]+output_dict["navigation-transportation"]+output_dict["abstract"]+output_dict["app"]+output_dict["web"]+output_dict["infographics"]+output_dict["misc"])/8 | |
| return output_dict | |