| from typing import List, Optional | |
| from mmengine import ConfigDict | |
| from opencompass.utils import dataset_abbr_from_cfg | |
| from opencompass.utils.prompt import get_prompt_hash | |
| from .default import DefaultSummarizer | |
| class CircularSummarizer(DefaultSummarizer): | |
| def __init__(self, | |
| config: ConfigDict, | |
| dataset_abbrs: Optional[List[str]] = None, | |
| summary_groups: List = [], | |
| prompt_db=None, | |
| metric_types=None) -> None: | |
| super().__init__(config, dataset_abbrs, summary_groups, prompt_db) | |
| self.metric_types = metric_types | |
| def _format_table(self, parsed_results, dataset_metrics, | |
| dataset_eval_mode): | |
| prompt_version = { | |
| dataset_abbr_from_cfg(d): get_prompt_hash(d)[:6] | |
| for d in self.dataset_cfgs | |
| } | |
| table = [] | |
| header1 = ['dataset', 'version', 'mode'] + sum( | |
| [[model_abbr] + ['-' for _ in range(len(self.metric_types) - 1)] | |
| for model_abbr in self.model_abbrs], []) | |
| table.append(header1) | |
| header2 = ['-', '-', '-'] + sum( | |
| [self.metric_types for _ in self.model_abbrs], []) | |
| table.append(header2) | |
| for dataset_abbr in self.dataset_abbrs: | |
| if dataset_abbr not in dataset_metrics: | |
| table.append([dataset_abbr, '-', '-'] + ['-'] * | |
| len(self.model_abbrs) * len(self.metric_types)) | |
| continue | |
| row = [ | |
| dataset_abbr, | |
| prompt_version.get(dataset_abbr, '-'), | |
| dataset_eval_mode.get(dataset_abbr, '-') | |
| ] | |
| for model_abbr in self.model_abbrs: | |
| for metric in self.metric_types: | |
| if dataset_abbr in parsed_results[ | |
| model_abbr] and metric in parsed_results[ | |
| model_abbr][dataset_abbr]: | |
| row.append('{:.02f}'.format( | |
| parsed_results[model_abbr][dataset_abbr][metric])) | |
| else: | |
| row.append('-') | |
| table.append(row) | |
| return table | |