import gradio as gr import os from transformers import AutoTokenizer from get_loss.get_loss_hf import run_get_loss import pdb from types import SimpleNamespace import pandas as pd import plotly.express as px import matplotlib.pyplot as plt import numpy as np from datetime import datetime import scipy # os.system('git clone https://github.com/EleutherAI/lm-evaluation-harness') # os.system('cd lm-evaluation-harness') # os.system('pip install -e .') # -i https://pypi.tuna.tsinghua.edu.cn/simple # 第一个功能：基于输入文本和对应的损失值对文本进行着色展示 def color_text(text_list=["hi", "FreshEval","!"], loss_list=[0.1,0.7]): """ 根据损失值为文本着色。 """ highlighted_text = [] # print('loss_list',loss_list) # ndarray to list loss_list = loss_list.tolist() loss_list=[0]+loss_list # print('loss_list',loss_list) # print('text_list',text_list) # pdb.set_trace() for text, loss in zip(text_list, loss_list): # color = "#FF0000" if float(loss) > 0.5 else "#00FF00" color=loss/20#TODO rescale # highlighted_text.append({"text": text, "bg_color": color}) highlighted_text.append((text, color)) print('highlighted_text',highlighted_text) return highlighted_text # 第二个功能：根据 ID 列表和 tokenizer 将 ID 转换为文本，并展示 def get_text(ids_list=[0.1,0.7], tokenizer=None): """ 给定一个 ID 列表和 tokenizer 名称，将这些 ID 转换成文本。 """ # return ['Hi', 'Adam'] # tokenizer = AutoTokenizer.from_pretrained(tokenizer) # print('ids_list',ids_list) # pdb.set_trace() text=[] for id in ids_list: text.append( tokenizer.decode(id, skip_special_tokens=True)) # 这里只是简单地返回文本，但是可以根据实际需求添加颜色或其他样式 print(f'L41:{text}') return text # def get_ids_loss(text, tokenizer, model): # """ # 给定一个文本，model and its tokenizer,返回其对应的 IDs 和损失值。 # """ # # tokenizer = AutoTokenizer.from_pretrained(tokenizer_name) # # model = AutoModelForCausalLM.from_pretrained(model_name) # # 这里只是简单地返回 IDs 和损失值，但是可以根据实际需求添加颜色或其他样式 # return [1, 2], [0.1, 0.7] def harness_eval(question, answer_index, answer_type, model=None,*choices,): ''' use harness to test one question, can specify the model, (extract or ppl) ''' # print(f'question,choices,answer_index,model,tokenizer: {question,choices,answer_index,model,tokenizer}') print(f'type of choices: {type(choices)} and type of choices[0]: {type(choices[0])}') print(f'choices: {choices}') # TODO add the model and its score # torch.nn.functional.softmax(output.logits, dim=0) # topk = torch.topk(output.logits, 5) return {'A':0.5, 'B':0.3, 'C':0.1, 'D':0.1} def plotly_plot_text():#(df, x, y, color,title, x_title, y_title): # plotly_plot(sample_df, 'date', 'loss_mean_at_1000', 'model','ppl with time', 'time', 'ppl') df=pd.read_csv('./data/tmp.csv') df['date'] = pd.to_datetime(df['date']) # sort by date df.sort_values(by='date', inplace=True) # use a dic to filter the dataframe df = df[df['file_name'] == 'arxiv_computer_science'] x,y,color,title, x_title, y_title='date', 'loss_mean_at_1000', 'model','ppl with time', 'time', 'ppl' fig = px.line(df, x=x, y=y, color=color,title=title) fig.update_xaxes(title_text=x_title) fig.update_yaxes(title_text=y_title) # fig.update_layout() return fig def plotly_plot_question(use_start=True,gjo=True,time_diff=False):#(df, x, y, color,title, x_title, y_title): # plotly_plot(sample_df, 'date', 'loss_mean_at_1000', 'model','ppl with time', 'time', 'ppl') data=pd.read_csv('./data/0309_merge_gjo.csv') # Model_x,Release Date,model,MMLU,GSM8,Humanities,SocialSciences,STEM,Other,Longbench,Question,Model_y,Start Time,End Time,Acc,Right Possibility data['date'] = pd.to_datetime(data['End Time']) # sort by date data.sort_values(by='date', inplace=True) # use a dic to filter the dataframe # df = df[df['file_name'] == 'arxiv_computer_science'] # x,y,color,title, x_title, y_title='date', 'Right Possibility', 'model','Right Possibility with time', 'time', 'Right Possibility' # fig = px.line(df, x=x, y=y, color=color,title=title) # fig.update_xaxes(title_text=x_title) # fig.update_yaxes(title_text=y_title) if not use_start: data['Start Time']=data['End Time'] # # Convert the 'Release Date' and 'Start Time' columns to datetime data['Release Date'] = pd.to_datetime(data['Release Date']) data['Start Time'] = pd.to_datetime(data['Start Time']) data_cleaned = data.dropna(subset=['Release Date', 'Start Time']) if time_diff: if gjo: data_cleaned['Time Difference (Months)'] = ((data_cleaned['Start Time'] - data_cleaned['Release Date']) / pd.Timedelta(days=90)).round().astype(int) else: data_cleaned['Time Difference (Months)'] = ((data_cleaned['Start Time'] - data_cleaned['Release Date']) / pd.Timedelta(days=365)).round().astype(int) else: time_point= datetime(2015, 1, 1) data_cleaned['Time Difference (Months)'] = ((data_cleaned['Start Time'] - time_point) / pd.Timedelta(days=90)).round().astype(int) # Step 1: Fill missing months with linear interpolation (if necessary) # Note: This dataset might not have explicit missing months, but we will ensure continuity for plotting # pdb.set_trace() # data_cleaned # data_cleaned['Time Difference (Months)'].value_counts() # Ensure 'Time Difference (Months)' is sorted for each model before applying rolling mean data_cleaned.sort_values(by=['Model_x', 'Time Difference (Months)'], inplace=True) import plotly.graph_objects as go from plotly.subplots import make_subplots import plotly.express as px from scipy.interpolate import CubicSpline # Initialize figure with subplots # fig = make_subplots(rows=2, cols=1, subplot_titles=('Accuracy (Acc)', 'Right Possibility')) # make this pic large enough fig = make_subplots(rows=2, cols=1, subplot_titles=('Accuracy (Acc)', 'Right Possibility'),vertical_spacing=0.1) colors = px.colors.qualitative.Plotly # Use Plotly's qualitative colors for consistency # Iterate over each unique model to plot their data for i, (model_name, group) in enumerate(data_cleaned.groupby('Model_x')): color = colors[i % len(colors)] # Cycle through colors # mean accuracy and right possibility for each model group=group.groupby(['Model_x', 'Time Difference (Months)'])\ .agg({'Acc':'mean','Right Possibility':'mean','Release Date':'first','Start Time':'first'}).reset_index() # Divide the data into before and after based on 'Release Date' and 'Start Time' before = group[group['Release Date'] >= group['Start Time']] after = group[group['Release Date'] < group['Start Time']] # Concat the last row of 'before' to 'after' if 'before' is not empty if not before.empty: after = pd.concat([before.iloc[[-1]], after]) # # ================================================================================ # before = CubicSpline(before['Time Difference (Months)'], before['Acc']) # after = CubicSpline(after['Time Difference (Months)'], after['Acc']) # before = CubicSpline(before['Time Difference (Months)'], before['Right Possibility']) # after = CubicSpline(after['Time Difference (Months)'], after['Right Possibility']) # # ================================================================================ # Plot 'Acc' on the first subplot fig.add_trace(go.Scatter(x=before['Time Difference (Months)'], y=before['Acc'], mode='lines', name=model_name + ' (Acc before)', line=dict(color=color)), row=1, col=1) fig.add_trace(go.Scatter(x=after['Time Difference (Months)'], y=after['Acc'], mode='lines', name=model_name + ' (Acc after)', line=dict(color=color, dash='dash')), row=1, col=1) # Plot 'Right Possibility' on the second subplot fig.add_trace(go.Scatter(x=before['Time Difference (Months)'], y=before['Right Possibility'], mode='lines', name=model_name + ' (Right Possibility before)', line=dict(color=color)), row=2, col=1) fig.add_trace(go.Scatter(x=after['Time Difference (Months)'], y=after['Right Possibility'], mode='lines', name=model_name + ' (Right Possibility after)', line=dict(color=color, dash='dash')), row=2, col=1) # Update layout if needed fig.update_layout(height=600, width=800, title_text="Model Performance Over Time") # fig.update_layout() return fig # def plotly_plot(df, x, y, color, title, x_title, y_title): # fig = px.line(df, x=x, y=y, color=color, title=title) # fig.update_xaxes(title_text=x_title) # fig.update_yaxes(title_text=y_title) # return fig def show_attention_plot(model_name,texts): # 初始化分词器和模型，确保在模型配置中设置 output_attentions=True args=SimpleNamespace(texts=texts,model=model_name) print(f'L60,text:{texts}') rtn_dic=run_get_loss(args) # print(rtn_dic) # pdb.set_trace() # {'logit':logit,'input_ids':input_chunk,'tokenizer':tokenizer,'neg_log_prob_temp':neg_log_prob_temp} # ids, loss =rtn_dic['input_ids'],rtn_dic['loss']#= get_ids_loss(text, tokenizer, model) # notice here is numpy ndarray tokenizer, model = rtn_dic['tokenizer'],rtn_dic['model'] text = "Here is some text to encode" # 使用分词器处理输入文本 inputs = tokenizer(text, return_tensors="pt") # 进行前向传播，获取输出 outputs = model(**inputs, output_attentions=True) # 检查是否成功获得了 attentions if "attentions" in outputs: last_layer_attentions = outputs.attentions[-1] # 获取最后一层的 attention 矩阵 print("Successfully retrieved the attention matrix:", last_layer_attentions.shape) else: pdb.set_trace() print("Attention matrix not found in outputs.") # 假设 last_layer_attentions 是我们从模型中提取的注意力矩阵 # last_layer_attentions 的形状应该是 [batch_size, num_heads, seq_length, seq_length] # 为了简化，我们这里只查看第一个样本、第一个头的注意力矩阵 attention_matrix = last_layer_attentions[0, 0].detach().numpy() # 使用 matplotlib 绘制热图 plt.figure(figsize=(10, 8)) plt.imshow(attention_matrix, cmap='viridis') # 添加标题和标签以提高可读性 plt.title('Attention Matrix Visualization') plt.xlabel('Tokens in Sequence') plt.ylabel('Tokens in Sequence') # 添加颜色条 plt.colorbar() # 保存图表到文件 # plt.savefig('/223040239/medbase/attention_matrix_visualization.png') return plt def color_pipeline(texts=["Hi","FreshEval","!"], model=None): """ 给定一个文本，返回其对应的着色文本。 """ print('text,model',texts,model) args=SimpleNamespace(texts=texts,model=model) print(f'L60,text:{texts}') rtn_dic=run_get_loss(args) # print(rtn_dic) # pdb.set_trace() # {'logit':logit,'input_ids':input_chunk,'tokenizer':tokenizer,'neg_log_prob_temp':neg_log_prob_temp} ids, loss =rtn_dic['input_ids'],rtn_dic['loss']#= get_ids_loss(text, tokenizer, model) # notice here is numpy ndarray tokenizer=rtn_dic['tokenizer'] # get tokenizer text = get_text(ids, tokenizer) # print('ids, loss ,text',ids, loss ,text) return color_text(text, loss) # TODO can this be global ? maybe need session to store info of the user # visible_btn_num = 4 # 创建 Gradio 界面 with gr.Blocks() as demo: # visible_btn_num = 4 model_input = gr.Textbox(label="model name", placeholder="input your model name here... now I am trying phi-2...")#TODO make a choice here with gr.Tab("color your text"): with gr.Row(): text_input = gr.Textbox(label="input text", placeholder="input your text here...") # file_input = gr.File(file_count="multiple",label='to add content')# # TODO craw and drop the file # loss_input = gr.Number(label="loss") # model_input = gr.Textbox(label="model name", placeholder="input your model name here... now I am trying phi-2...")#TODO make a choice here output_box=gr.HighlightedText(label="colored text")#,interactive=True gr.Examples( [ ["Hi FreshEval !", "microsoft/phi-2"], ["Hello FreshBench !", "/home/sribd/chenghao/models/phi-2"], ], [text_input, model_input],) # cache_examples=True, # # cache_examples=False, # fn=color_pipeline, # outputs=output_box # ) # TODO select models that can be used online # TODO maybe add our own models color_text_output = gr.HTML(label="colored text") color_text_button = gr.Button("color the text").click(color_pipeline, inputs=[text_input, model_input], outputs=output_box) # markdown gr.Markdown('### How to use this app') attention_plot=gr.Plot(label='attention plot') see_attention_button = gr.Button("see attention").click(show_attention_plot,inputs=[model_input, text_input],outputs=[attention_plot]) date_time_input = gr.Textbox(label="the date when the text is generated")#TODO add date time input description_input = gr.Textbox(label="description of the text") submit_button = gr.Button("submit a post or record").click() #TODO add model and its score with gr.Tab('test your qeustion'): ''' use extract, or use ppl ''' question=gr.Textbox(label="input question", placeholder='input your question here...') answer_index=gr.Textbox(label="right answer index", placeholder='index for right anser here, start with 0')#TODO add multiple choices, # model_input = gr.Textbox(label="model name", placeholder="input your model name here... now I am trying phi-2...")#TODO make a choice here btn_list = [] # choices=gr.Textbox(placeholder='input your other choices here...') button_limit=10 # global visible_btn_num visible_btn_num = 4 from gradio_samples.add_components import add_one_btn, remove_one_btn, get_text_content # use partial function from functools import partial add_one_btn=partial(add_one_btn,button_limit=button_limit,)#visible_btn_num = visible_btn_num) remove_one_btn=partial(remove_one_btn,button_limit=button_limit,)#visible_btn_num = visible_btn_num) # with gr.Blocks() as demo: with gr.Row(): for i in range(button_limit): if i