okay on local phi-2

app.py

@@ -1,25 +1,28 @@
 import gradio as gr
 import os
 from transformers import AutoTokenizer
-from .get_loss.get_loss_hf import run_get_loss
-
+from get_loss.get_loss_hf import run_get_loss
+import pdb
+from types import SimpleNamespace
 # os.system('git clone https://github.com/EleutherAI/lm-evaluation-harness')
 # os.system('cd lm-evaluation-harness')
 # os.system('pip install -e .')
+# -i https://pypi.tuna.tsinghua.edu.cn/simple
 # 第一个功能：基于输入文本和对应的损失值对文本进行着色展示
 
-def color_text(text_list=["hi", "FreshEval"], loss_list=[0.1,0.7]):
+def color_text(text_list=["hi", "FreshEval","!"], loss_list=[0.1,0.7]):
     """
     根据损失值为文本着色。
     """
     highlighted_text = []
+    loss_list=[0]+loss_list
     for text, loss in zip(text_list, loss_list):
         # color = "#FF0000" if float(loss) > 0.5 else "#00FF00"
-        color=loss
+        color=loss/25
         # highlighted_text.append({"text": text, "bg_color": color})
         highlighted_text.append((text, color))
 
-        print(highlighted_text)
+        print('highlighted_text',highlighted_text)
     return highlighted_text
 
 # 第二个功能：根据 ID 列表和 tokenizer 将 ID 转换为文本，并展示
@@ -27,32 +30,43 @@ def get_text(ids_list=[0.1,0.7], tokenizer=None):
     """
     给定一个 ID 列表和 tokenizer 名称，将这些 ID 转换成文本。
     """
-    return ['Hi', 'Adam']
-    # tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
-    # text = tokenizer.decode(eval(ids_list), skip_special_tokens=True)
+    # return ['Hi', 'Adam']
+    # tokenizer = AutoTokenizer.from_pretrained(tokenizer)
+    print('ids_list',ids_list)
+    # pdb.set_trace()
+    text=[]
+    for id in ids_list:
+        text.append(  tokenizer.decode(id, skip_special_tokens=True))
     # 这里只是简单地返回文本，但是可以根据实际需求添加颜色或其他样式
-    # return text
+    print(f'L41:{text}')
+    return text
 
 
-def get_ids_loss(text, tokenizer, model):
-    """
-    给定一个文本，model and its tokenizer,返回其对应的 IDs 和损失值。
-    """
-    # tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
-    # model = AutoModelForCausalLM.from_pretrained(model_name)
-    # 这里只是简单地返回 IDs 和损失值，但是可以根据实际需求添加颜色或其他样式
-    return [1, 2], [0.1, 0.7]
+# def get_ids_loss(text, tokenizer, model):
+#     """
+#     给定一个文本，model and its tokenizer,返回其对应的 IDs 和损失值。
+#     """
+#     # tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
+#     # model = AutoModelForCausalLM.from_pretrained(model_name)
+#     # 这里只是简单地返回 IDs 和损失值，但是可以根据实际需求添加颜色或其他样式
+#     return [1, 2], [0.1, 0.7]
 
 
-def color_pipeline(text=["hi", "FreshEval"],  model=None):
+def color_pipeline(texts=["Hi","FreshEval","!"],  model=None):
     """
     给定一个文本，返回其对应的着色文本。
     """
-    # rtn_dic=run_get_loss()
+    print('text,model',texts,model)
+    args=SimpleNamespace(texts=texts,model=model)
+    print(f'L60,text:{texts}')
+    rtn_dic=run_get_loss(args)
+    # print(rtn_dic)
+    # pdb.set_trace()
     # {'logit':logit,'input_ids':input_chunk,'tokenizer':tokenizer,'neg_log_prob_temp':neg_log_prob_temp}
-    tokenizer=None # get tokenizer
-    ids, loss = get_ids_loss(text, tokenizer, model)
+    ids, loss =rtn_dic['input_ids'],rtn_dic['loss']#= get_ids_loss(text, tokenizer, model)
+    tokenizer=rtn_dic['tokenizer'] # get tokenizer
     text = get_text(ids, tokenizer)
+    # print('ids, loss ,text',ids, loss ,text)
     return color_text(text, loss)
 
 
@@ -67,20 +81,25 @@ with gr.Blocks() as demo:
 
             # loss_input = gr.Number(label="loss")
             model_input = gr.Textbox(label="model name", placeholder="input your model name here... now I am trying phi-2...")
+            output_box=gr.HighlightedText(label="colored text")
+            # gr.Examples(
+            #     [
+            #         # ["Hi FreshEval !", "microsoft/phi-2"],
+            #         ["Hello FreshBench !", "/home/sribd/chenghao/models/phi-2"],
+            #     ],
+            #     [text_input, model_input],
+            #     cache_examples=True,
+            #     # cache_examples=False,
+            #     fn=color_pipeline,
+            #     outputs=output_box
+            # )
             # TODO select models that can be used online
             # TODO maybe add our own models
 
 
             color_text_output = gr.HTML(label="colored text")
-            # gr.Markdown("## Text Examples")
-            # gr.Examples(
-            #     [["hi", "Adam"], [0.1,0.7]],
-            #     [text_input, loss_input],
-            #     cache_examples=True,
-            #     fn=color_text,
-            #     outputs=color_text_output
-            # )
-        color_text_button = gr.Button("color the text").click(color_pipeline, inputs=[text_input, model_input], outputs=gr.HighlightedText(label="colored text"))
+
+        color_text_button = gr.Button("color the text").click(color_pipeline, inputs=[text_input, model_input], outputs=output_box)
 
 
         date_time_input = gr.Textbox(label="the date when the text is generated")#TODO add date time input

get_loss/get_loss.py

@@ -257,7 +257,7 @@ def run_get_loss(args):
 from types import SimpleNamespace
 
 if __name__ == '__main__':
-    args=SimpleNamespace(model='microsft/phi-2',model_type='hf',data='data.json',log_path='./logs/',model_cache=None,chunk_size=1024)
+    args=SimpleNamespace(model='microsoft/phi-2',texts=['Hello FreshBench !'],model_type='hf',data='data.json',model_cache=None,chunk_size=1024)
 
 
 

get_loss/get_loss_hf.py

@@ -10,13 +10,15 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
 from datetime import datetime
 import argparse
 from types import SimpleNamespace
-
+import pdb
 # import mamba_ssm
 # import rwkv
 
 
 # RWKV4_TOKENIZER_FILE = "./support/20B_tokenizer.json"
-device = 'cuda' if torch.cuda.is_available() else 'cpu'
+# device = 'cuda' if torch.cuda.is_available() else 'cpu'
+device = 'cpu'
+
 
 def load_list_from_json(file_path):
     """
@@ -36,6 +38,7 @@ def calculate_loss(logits, target_token_ids):
     # log_probs = F.log_softmax(shifted_logits, dim=-1)
     loss = torch.nn.functional.cross_entropy(logits[:-1, :].view(-1, logits.shape[-1]), 
                                              target_token_ids[1:].view(-1), reduction='none')
+    # pdb.set_trace()
 
 
     # target_log_probs = -log_probs.gather(1, shifted_targets.unsqueeze(1)).squeeze()
@@ -44,7 +47,7 @@ def calculate_loss(logits, target_token_ids):
     # log_sum = torch.sum(target_log_probs, dim=-1)
     # print(perplexity_sum)
 
-    return loss.item()
+    return loss.cpu().numpy()
 
 
 def calculate_log_sum(logits, target_token_ids):
@@ -122,6 +125,7 @@ def print_model_parameters_in_billions(model):
 def load_hf_model(path, cache_path):
     hf_tokenizer = AutoTokenizer.from_pretrained(path)
     if cache_path is not None:
+        # pdb.set_trace()
         hf_model = AutoModelForCausalLM.from_pretrained(path,
                                                         device_map=device,
                                                         trust_remote_code=True,
@@ -212,7 +216,6 @@ def eval_hf_model(model, tokenizer, texts, chunk_size):
         neg_log_prob_temp += log_sum
 
         loss = calculate_loss(logit, input_chunk.squeeze(0))
-        neg_log_prob_temp += log_sum
 
     #         token_length_list.append(seq_length)
     #         data.append(neg_log_prob_temp)
@@ -224,8 +227,8 @@ def eval_hf_model(model, tokenizer, texts, chunk_size):
 
     # print(f'log probability sum: {sum(data) / len(data):.2f}')
     # print(f'avg tokens: {sum(token_length_list) / len(token_length_list):.0f}')
-    rtn_dic={'logit':logit,'input_ids':input_chunk,'loss':loss,'tokenizer':tokenizer,'neg_log_prob_temp':neg_log_prob_temp}
-    return rtn_dic
+        rtn_dic={'logit':logit.cpu().numpy(),'input_ids':input_chunk.cpu().numpy()[0],'loss':loss,'tokenizer':tokenizer,'neg_log_prob_temp':neg_log_prob_temp}
+        return rtn_dic
 
 
 # if __name__ == '__main__':
@@ -239,14 +242,28 @@ def eval_hf_model(model, tokenizer, texts, chunk_size):
 #     parser.add_argument('--chunk_size', type=int, default=1024, help='chunk size')
 
 
-def run_get_loss(args):
+def run_get_loss(args=None):
     if args is None:
-        args=SimpleNamespace(model='microsoft/phi-2',texts='Hello FreshBench !',model_type='hf',model_cache=None,chunk_size=1024)
+        # args=SimpleNamespace(model='microsoft/phi-2',texts='Hello FreshBench !',model_type='hf',model_cache=None,chunk_size=1024)
+        args=SimpleNamespace(model='/home/sribd/chenghao/models/phi-2',texts='Hello FreshBench !',model_type='hf',model_cache=None,chunk_size=1024)
+
+    
+    if 'chunk_size' not in args.__dict__:
+        args.chunk_size=1024
+    if 'model_type' not in args.__dict__:
+        args.model_type='hf'
+    if 'model' not in args.__dict__ or len(args.model)<2:
+        # args.model='/home/sribd/chenghao/models/phi-2'
+        args.model='microsoft/phi-2'
+
+    if 'model_cache' not in args.__dict__:
+        args.model_cache=args.model
 
     # args = parser.parse_args()
 
     # load data
     # texts = load_list_from_json(args.data)
+    print('args',args)
     texts=args.texts
     print(f'data size: {len(texts)}')
 
@@ -264,6 +281,7 @@ def run_get_loss(args):
 
     # eval
     if args.model_type in ['hf', 'mamba']:
+        print(f'eval hf')
         return eval_hf_model(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size)
     # elif args.model_type == 'rwkv':
     #     return eval_rwkv(model=model, tokenizer=tokenizer, texts=texts, chunk_size=args.chunk_size)

gradio_cached_examples/186/log.csv

@@ -0,0 +1,3 @@
+colored text,flag,username,timestamp
+"[{""token"":""Hi"",""class_or_confidence"":13.59826946258545},{""token"":""Adam"",""class_or_confidence"":14.804081916809082}]",,,2024-03-14 14:05:40.149274
+"[{""token"":""Hi"",""class_or_confidence"":13.59826946258545},{""token"":""Adam"",""class_or_confidence"":14.804081916809082}]",,,2024-03-14 14:05:42.364248

gradio_cached_examples/212/log.csv

@@ -0,0 +1,3 @@
+colored text,flag,username,timestamp
+"[{""token"":""Hi"",""class_or_confidence"":13.59826946258545},{""token"":""Adam"",""class_or_confidence"":14.804081916809082}]",,,2024-03-14 14:05:44.632048
+"[{""token"":""Hi"",""class_or_confidence"":13.59826946258545},{""token"":""Adam"",""class_or_confidence"":14.804081916809082}]",,,2024-03-14 14:05:46.813954