Update app.py

app.py

@@ -2,40 +2,69 @@ from transformers import AutoTokenizer
 import gradio as gr
 
 
-# Define tokenizer models
-MODELS = {
-    "LLaMa-1/LLaMa-2": "TheBloke/Llama-2-7B-fp16",
-    "LLaMa-3": "unsloth/llama-3-8b",
-    "Mistral": "mistral-community/Mistral-7B-v0.2",
-    "GPT-2/GPT-J": "openai-community/gpt2",
-    "GPT-NeoX": "EleutherAI/gpt-neox-20b",
-    "Falcon": "tiiuae/falcon-7b",
-    "Phi-1/Phi-2": "microsoft/phi-2",
-    "Phi-3": "microsoft/Phi-3-mini-4k-instruct",
-    "T5": "google/flan-t5-xxl",
-    "Gemma": "alpindale/gemma-2b",
-    "Command-R": "CohereForAI/c4ai-command-r-plus",
-    "Qwen/Qwen1.5": "Qwen/Qwen1.5-7B",
-    "CodeQwen": "Qwen/CodeQwen1.5-7B",
-    "RWKV-v4": "RWKV/rwkv-4-14b-pile",
-    "RWKV-v5/RWKV-v6": "RWKV/v5-EagleX-v2-7B-HF",
-    "DeepSeek-LLM": "deepseek-ai/deepseek-llm-7b-base",
-    "DeepSeek-V2": "deepseek-ai/DeepSeek-V2"
-}
+def tokenize(input_text):
+    llama_tokens = len(llama_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    llama3_tokens = len(llama3_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    mistral_tokens = len(mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gpt2_tokens = len(gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gpt_neox_tokens = len(gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    falcon_tokens = len(falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    phi2_tokens = len(phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    phi3_tokens = len(phi3_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    t5_tokens = len(t5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gemma_tokens = len(gemma_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    command_r_tokens = len(command_r_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    qwen_tokens = len(qwen_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    codeqwen_tokens = len(codeqwen_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    rwkv4_tokens = len(rwkv4_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    rwkv5_tokens = len(rwkv5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    deepseekllm_tokens = len(deepseekllm_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    deepseekv2_tokens = len(deepseekv2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
 
+    results = {
+        "LLaMa-1/LLaMa-2": llama_tokens,
+        "LLaMa-3": llama3_tokens,
+        "Mistral": mistral_tokens,
+        "GPT-2/GPT-J": gpt2_tokens,
+        "GPT-NeoX": gpt_neox_tokens,
+        "Falcon": falcon_tokens,
+        "Phi-1/Phi-2": phi2_tokens,
+        "Phi-3": phi3_tokens,
+        "T5": t5_tokens,
+        "Gemma": gemma_tokens,
+        "Command-R": command_r_tokens,
+        "Qwen/Qwen1.5": qwen_tokens,
+        "CodeQwen": codeqwen_tokens,
+        "RWKV-v4": rwkv4_tokens,
+        "RWKV-v5/RWKV-v6": rwkv5_tokens,
+        "DeepSeek-LLM": deepseekllm_tokens,
+        "DeepSeek-V2": deepseekv2_tokens
+    }
 
-def tokenize(input_text):
-    results = {}
-    for model_name, model_tokenizer in MODELS.items():
-        tokenizer = AutoTokenizer.from_pretrained(model_tokenizer, trust_remote_code=True)
-        tokens = len(tokenizer(input_text, add_special_tokens=True)["input_ids"])
-        results[model_name] = tokens
-    
     # Sort the results in descending order based on token length
     sorted_results = sorted(results.items(), key=lambda x: x[1], reverse=True)
+
     return "\n".join([f"{model}: {tokens}" for model, tokens in sorted_results])
 
 
 if __name__ == "__main__":
-    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(label="Input Text", lines=len(MODELS)), outputs="text")
-    iface.launch()
+    llama_tokenizer = AutoTokenizer.from_pretrained("TheBloke/Llama-2-7B-fp16")
+    llama3_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-3-8b")
+    mistral_tokenizer = AutoTokenizer.from_pretrained("mistral-community/Mistral-7B-v0.2")
+    gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
+    gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
+    falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
+    phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
+    phi3_tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
+    t5_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl")
+    gemma_tokenizer = AutoTokenizer.from_pretrained("alpindale/gemma-2b")
+    command_r_tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-plus")
+    qwen_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen1.5-7B")
+    codeqwen_tokenizer = AutoTokenizer.from_pretrained("Qwen/CodeQwen1.5-7B")
+    rwkv4_tokenizer = AutoTokenizer.from_pretrained("RWKV/rwkv-4-14b-pile", trust_remote_code=True)
+    rwkv5_tokenizer = AutoTokenizer.from_pretrained("RWKV/v5-EagleX-v2-7B-HF", trust_remote_code=True)
+    deepseekllm_tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/deepseek-llm-7b-base", trust_remote_code=True)
+    deepseekv2_tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V2", trust_remote_code=True)
+
+    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(label="Input Text", lines=17), outputs="text")
+    iface.launch()