File size: 2,381 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
# flake8: noqa: E501
import csv
import os
import os.path as osp
import re
from collections import defaultdict
from datetime import datetime
import numpy as np
from mmengine import ConfigDict
try:
from prettytable import from_csv
except ImportError:
from_csv = None
from opencompass.utils import model_abbr_from_cfg
from .alignmentbench import AlignmentBenchSummarizer, post_process_alignbench
from .subjective_post_process import post_process_autoj, post_process_judgelm
from .utils import get_judgeanswer_and_reference, get_outdir
CATEGORIES = {
'中文': ['内容扩写_ZH', '内容续写_ZH', '内容改写_ZH'],
'英文': ['内容扩写_EN', '内容续写_EN', '内容改写_EN'],
}
All_Dimensions = [
'Creativity', 'Richness', 'User Demand Fulfillment', 'Logical Coherence',
'Overall Score', '创造性', '丰富度', '满足用户需求', '逻辑连贯性', '综合得分'
]
def post_process_creationbench(judgement: str,
all_dimensions=All_Dimensions,
possible_keys=['综合得分', 'Overall Score']):
"""Input a string like below:
xxx{'事实正确性': 1, '满足用户需求': 1, '清晰度': 2, '完备性': 1, '综合得分': 1}xxx,
and extract each score
"""
return post_process_alignbench(judgement, all_dimensions, possible_keys)
class CreationBenchSummarizer(AlignmentBenchSummarizer):
"""Do the subjectivity analyze based on evaluation results.
Args:
config (ConfigDict): The configuration object of the evaluation task.
It's expected to be filled out at runtime.
"""
def __init__(self, config: ConfigDict, judge_type: str) -> None:
super().__init__(config, judge_type)
self.judge_map = {
'general': post_process_creationbench,
'autoj': post_process_autoj,
'judgelm': post_process_judgelm
}
self.judge_function = self.judge_map[self.judge_type]
self.category = CATEGORIES
def summarize(self,
time_str: str = datetime.now().strftime('%Y%m%d_%H%M%S')):
"""Summarize the subjectivity analysis based on evaluation results.
Args:
time_str (str): Timestamp for file naming.
Returns:
pd.DataFrame: The summary results.
"""
super().summarize(time_str)
|