import json import os import glob import argparse import csv def chatgpt_json(merge_file): # chat results merge_data = merge_file.decode("utf-8") merge_data = merge_data.replace(": true,", ": \"true\",") merge_data = merge_data.replace(": false,", ": \"false\",") merge_data = eval(merge_data) dataset_scores_dict = {} for dataset_name, dataset_results in merge_data.items(): correct, total_nums = 0, 0 for id in dataset_results: for dim in dataset_results[id]: for result in dataset_results[id][dim]: correct += result['rating'] total_nums += 1 dataset_scores_dict[dataset_name] = round(correct / total_nums * 100, 2) # dataset_scores_dict[dataset_name] = round(correct / total_nums , 4) return dataset_scores_dict def compute_scores(merge_file): merge_data = merge_file.decode("utf-8") merge_data = merge_data.replace(": true,", ": \"true\",") merge_data = merge_data.replace(": false,", ": \"false\",") merge_data = merge_data.replace(": null,", ": \"null\",") merge_data = eval(merge_data) dataset_scores_dict = {} total_correct, total_num = 0, 0 eval_dims = ['action', 'speed', 'direction', 'order', 'attribute_change', 'avg'] dim_correct, dim_total = {dim: 0 for dim in eval_dims if dim!='avg'}, {dim: 0 for dim in eval_dims if dim!='avg'} for dataset_name, dataset_results in merge_data.items(): dataset_correct, dataset_num = {dim: 0 for dim in eval_dims}, {dim: 0 for dim in eval_dims} for id in dataset_results: for dim in dataset_results[id]: for result in dataset_results[id][dim]: dataset_correct['avg'] += result['rating'] dataset_correct[dim] += result['rating'] dim_correct[dim] += result['rating'] dataset_num['avg'] += 1 dataset_num[dim] += 1 dim_total[dim] += 1 total_correct += dataset_correct['avg'] total_num += dataset_num['avg'] for dim in eval_dims: dataset_scores_dict[f"{dim}_{dataset_name}"] = round(dataset_correct[dim] / dataset_num[dim] * 100, 2) for dim in dim_correct: dataset_scores_dict[f"avg_{dim}"] = round(dim_correct[dim] / dim_total[dim] * 100, 2) dataset_scores_dict["avg_all"] = round(total_correct / total_num * 100, 2) # print(dataset_score_dict) # with open(args.score_output_file, 'w', encoding='utf-8') as f: # json.dump(dataset_score_dict, f, indent=2) # print(f'{args.score_output_file} is saved!') # ======================== data = [ ["Avg. All", "Avg. Action", "Avg. Direction", "Avg. Speed", "Avg. Event Order", "Avg. Attribute Change", "Avg. Multi-Choice", "Avg. Yes/No", "Avg. Caption Matching", "Avg. Caption Generation", "Action. Multi-Choice", "Action. Yes/No", "Action. Caption Matching", "Action. Caption Generation", "Direction. Multi-Choice", "Direction. Yes/No", "Direction. Caption Matching", "Direction. Caption Generation", "Speed. Multi-Choice", "Speed. Yes/No", "Speed. Caption Matching", "Speed. Caption Generation", "Event Order. Multi-Choice", "Event Order. Yes/No", "Event Order. Caption Matching", "Event Order. Caption Generation", "Attribute Change. Multi-Choice", "Attribute Change. Yes/No", "Attribute Change. Caption Matching", "Attribute Change. Caption Generation"], [dataset_scores_dict["avg_all"], dataset_scores_dict["avg_action"], dataset_scores_dict["avg_direction"], dataset_scores_dict["avg_speed"], dataset_scores_dict["avg_order"], dataset_scores_dict["avg_attribute_change"], dataset_scores_dict["avg_multi-choice"], dataset_scores_dict["avg_yes_no"], dataset_scores_dict["avg_caption_matching"], dataset_scores_dict["avg_captioning"], dataset_scores_dict['action_multi-choice'], dataset_scores_dict['action_yes_no'], dataset_scores_dict['action_caption_matching'], dataset_scores_dict['action_captioning'], dataset_scores_dict['direction_multi-choice'], dataset_scores_dict['direction_yes_no'], dataset_scores_dict['direction_caption_matching'], dataset_scores_dict['direction_captioning'], dataset_scores_dict['speed_multi-choice'], dataset_scores_dict['speed_yes_no'], dataset_scores_dict['speed_caption_matching'], dataset_scores_dict['speed_captioning'], dataset_scores_dict['order_multi-choice'], dataset_scores_dict['order_yes_no'], dataset_scores_dict['order_caption_matching'], dataset_scores_dict['order_captioning'], dataset_scores_dict['attribute_change_multi-choice'], dataset_scores_dict['attribute_change_yes_no'], dataset_scores_dict['attribute_change_caption_matching'], dataset_scores_dict['attribute_change_captioning'], ], ] return data