Refactor code and optimize dataframe sorting

app.py

@@ -6,10 +6,6 @@ from datasets import load_dataset
 import random
 from pathlib import Path
 
-# tokenizer = AutoTokenizer.from_pretrained("Xenova/gpt-4o")
-# token_ids = tokenizer.encode("السلام عليكم ورحمة الله")
-# exit()
-
 initial_list_of_models = [
     "Xenova/gpt-4o",
     "NousResearch/Meta-Llama-3-8B",
@@ -48,16 +44,12 @@ for model_name in tqdm(initial_list_of_models):
 df = df.sort_values(by="➕ Total Number of Tokens", ascending=True)
 
 # Save the dataframe to a csv file
-df.to_json(dataframe_path, lines=True, orient="records")
-
-# Gradio Functions
-def refresh():
-    global df
-    df = df.sort_values(by="➕ Total Number of Tokens", ascending=True)
-    return gr.Dataframe(df), gr.BarPlot(df), gr.Dropdown(choices=df["📛 Models"].tolist())
+df.to_json(dataframe_path, lines=True, orient="records", force_ascii=False)
 
 def submit(model_name):
     global df
+    if model_name in df["📛 Models"].values:
+        return gr.Dataframe(df), gr.BarPlot(df), gr.Dropdown(choices=df["📛 Models"].tolist())
     tokenizer = AutoTokenizer.from_pretrained(
         model_name, use_fast=True, trust_remote_code=True
     )
@@ -72,6 +64,9 @@ def submit(model_name):
         },
         ignore_index=True,
     )
+    df = df.sort_values(by="➕ Total Number of Tokens", ascending=True)
+    df.to_json(dataframe_path, lines=True, orient="records", force_ascii=False)
+    return gr.Dataframe(df), gr.BarPlot(df), gr.Dropdown(choices=df["📛 Models"].tolist())
 
 def generate_distinct_colors(n):
     """Generate n visually distinct colors in hexadecimal format."""
@@ -196,9 +191,7 @@ with gr.Blocks() as demo:
         model_name = gr.Textbox(
             label="Model Name from Hugging Face (e.g. Xenova/gpt-4o)"
         )
-        with gr.Row():
-            refresh_btn = gr.Button(value="Refresh")
-            submit_new_model_btn = gr.Button(value="Submit", variant="primary")
+        submit_new_model_btn = gr.Button(value="Submit", variant="primary")
     with gr.Tab(label="Try tokenizers"):
         text = gr.Textbox(label="Enter a text", lines=5, value="السلام عليكم ورحمة الله", rtl=True, text_align="right")
         dropdown = gr.Dropdown(
@@ -209,8 +202,7 @@ with gr.Blocks() as demo:
         submit_text_btn = gr.Button(value="Submit", variant="primary")
         tokenized_textbox = gr.HighlightedText(label="Tokenized text")
 
-    submit_new_model_btn.click(submit, model_name)
-    refresh_btn.click(refresh, outputs=[dataframe, barplot, dropdown])
+    submit_new_model_btn.click(submit, model_name, outputs=[dataframe, barplot, dropdown])
     submit_text_btn.click(tokenize_text, inputs=[text, dropdown], outputs=[tokenized_textbox])