Update app.py

app.py

@@ -19,6 +19,8 @@ def tokenize(input_text):
     rwkv4_tokens = len(rwkv4_tokenizer(input_text, add_special_tokens=True)["input_ids"])
     rwkv5_tokens = len(rwkv5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
     deepseek_tokens = len(deepseek_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    internlm_tokens = len(internlm_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    internlm2_tokens = len(internlm2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
 
     results = {
         "LLaMa-1/LLaMa-2": llama_tokens,
@@ -36,7 +38,9 @@ def tokenize(input_text):
         "CodeQwen": codeqwen_tokens,
         "RWKV-v4": rwkv4_tokens,
         "RWKV-v5/RWKV-v6": rwkv5_tokens,
-        "DeepSeek-LLM/DeepSeek-V2": deepseek_tokens
+        "DeepSeek-LLM/DeepSeek-V2": deepseek_tokens,
+        "InternLM": internlm_tokens,
+        "InternLM2": internlm_tokens
     }
 
     # Sort the results in descending order based on token length
@@ -62,6 +66,8 @@ if __name__ == "__main__":
     rwkv4_tokenizer = AutoTokenizer.from_pretrained("RWKV/rwkv-4-14b-pile", trust_remote_code=True)
     rwkv5_tokenizer = AutoTokenizer.from_pretrained("RWKV/v5-EagleX-v2-7B-HF", trust_remote_code=True)
     deepseek_tokenizer = AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V2", trust_remote_code=True)
+    internlm_tokenizer = AutoTokenizer.from_pretrained("internlm/internlm-20b", trust_remote_code=True)
+    internlm2_tokenizer = AutoTokenizer.from_pretrained("internlm/internlm2-20b", trust_remote_code=True)
 
-    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(label="Input Text", lines=17), outputs="text")
+    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(label="Input Text", lines=19), outputs="text")
     iface.launch()