import json import os import logging from datetime import datetime from lm_eval import tasks, evaluator, utils from src.envs import API from src.backend.manage_requests import EvalRequest logging.getLogger("openai").setLevel(logging.WARNING) def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, local_dir: str, results_repo: str, no_cache=True, limit=None): if limit: print( "!!!!!! WARNING: NOT A FULL EVALUATION !!!!! Just looking at %d." "The `LIMIT` environment variable and/or `limit` in `run_evaluation` " "SHOULD ONLY BE USED FOR TESTING. " "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT." % limit ) task_names = utils.pattern_match(task_names, tasks.ALL_TASKS) print(f"Selected Tasks: {task_names}") results = evaluator.simple_evaluate( model="hf-causal-experimental", # "hf-causal" model_args=eval_request.get_model_args(), tasks=task_names, num_fewshot=num_fewshot, batch_size=batch_size, device=device, no_cache=no_cache, limit=limit, write_out=True, output_base_path="logs" ) results["config"]["model_dtype"] = eval_request.precision results["config"]["model_name"] = eval_request.model results["config"]["model_sha"] = eval_request.revision dumped = json.dumps(results, indent=2) print(dumped) output_path = os.path.join(local_dir, *eval_request.model.split("/"), f"results_{datetime.now()}.json") os.makedirs(os.path.dirname(output_path), exist_ok=True) with open(output_path, "w") as f: f.write(dumped) print(evaluator.make_table(results)) API.upload_file( path_or_fileobj=output_path, path_in_repo=f"{eval_request.model}/results_{datetime.now()}.json", repo_id=results_repo, repo_type="dataset", ) return results