adding app

app.py

@@ -0,0 +1,135 @@
+import os
+import json
+import subprocess
+import torch
+import spaces
+import gradio as gr
+from threading import Thread
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from huggingface_hub import HfApi
+from datetime import datetime
+
+MODEL_ID = os.environ.get("MODEL_ID")
+DATASET_REPO = os.environ.get("DATASET_REPO") 
+DESCRIPTION = os.environ.get("DESCRIPTION")
+PROMPT = os.environ.get("PROMPT")
+
+ON_LOAD="""
+async()=>{
+    alert("Before using the service, users must agree to the following terms:\\n\\nPlease note that the model presented here is an experimental tool that is still being developed and improved.\\n\\nMeasures have been taken during the model creation process to minimizing the risk of generating vulgar, prohibited or inappropriate content. However, in rare cases, unwanted content may be generated. If you encounter any content that is deemed inappropriate or violates our policies, please contact us to report it. Your information will enable us to take further steps to improve and develop the model to make it safe and user-friendly.\\n\\nYou must not use the model for illegal, harmful, violent, racist or sexual purposes. Please do not send any private information. The website collects user dialogue data and reserves the right to distribute it under the Creative Commons Attribution (CC-BY) or similar license.");
+}
+"""
+
+api = HfApi()
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+
+@spaces.GPU()
+def generate(instruction, stop_tokens, repetition_penalty, max_new_tokens):
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    input_ids = tokenizer.apply_chat_template(instruction, return_tensors="pt", add_generation_prompt=True,).to(device)
+
+    if input_ids.shape[1] > 4096:
+        input_ids = input_ids[:, -4096:]
+
+    generate_kwargs = dict(
+        input_ids = input_ids,
+        streamer=streamer,
+        do_sample=False,
+        max_new_tokens=max_new_tokens,
+        repetition_penalty=repetition_penalty,
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+    outputs = []
+    for new_token in streamer:
+        if new_token in stop_tokens:
+            break
+        outputs.append(new_token.replace("<|im_end|>", ""))
+        yield "".join(outputs)
+
+
+def predict(message, history):
+    system_prompt = PROMPT
+    repetition_penalty = 1.1
+    max_new_tokens = 1024
+    stop_tokens = ["<|endoftext|>", "<|im_end|>"]
+    conversation = []
+    conversation.append(
+        {
+            "role": "system",
+            "content": system_prompt,
+        }
+    )
+    for user, assistant in history:
+        conversation.extend([{"role": "user", "content": user}, 
+                             {"role": "assistant", "content": assistant}])
+    conversation.append({"role": "user", "content": message})
+    print(conversation)
+    
+    for output_text in generate(conversation, stop_tokens, repetition_penalty, max_new_tokens):
+        if output_text in stop_tokens:
+            break
+        yield output_text
+
+    hfapi = HfApi()
+    day=datetime.now().strftime("%Y-%m-%d")
+    timestamp=datetime.now().timestamp()
+    dd={
+        'message': message, 
+        'history': history, 
+        'system_prompt':system_prompt, 
+        'max_new_tokens':max_new_tokens, 
+        'repetition_penalty':repetition_penalty, 
+        'instruction':conversation, 
+        'output':output_text,
+        'precision': 'auto '+str(model.dtype),
+    }
+    hfapi.upload_file(
+        path_or_fileobj=json.dumps(dd, indent=2, ensure_ascii=False).encode('utf-8'),
+        path_in_repo=f"{day}/{timestamp}.json",
+        repo_id=DATASET_REPO,
+        repo_type="dataset",
+        commit_message=f"X",
+        run_as_future=True
+    )
+
+def vote(chatbot, data: gr.LikeData):
+    day=datetime.now().strftime("%Y-%m-%d")
+    timestamp=datetime.now().timestamp()
+    api.upload_file(
+        path_or_fileobj=json.dumps({"history":chatbot, 'index': data.index, 'liked': data.liked}, indent=2, ensure_ascii=False).encode('utf-8'),
+        path_in_repo=f"liked/{day}/{timestamp}.json",
+        repo_id=DATASET_REPO,
+        repo_type="dataset",
+        commit_message=f"L",
+        run_as_future=True
+    )
+        
+with gr.Blocks(js=ON_LOAD) as demo:
+    chatbot = gr.Chatbot(label="Chatbot", likeable=True, render=False)
+    chatbot.like(vote, [chatbot], None)
+    gr.ChatInterface(
+        predict,
+        chatbot=chatbot,
+        title="MKLLM-7B-Instruct",
+        description=DESCRIPTION,
+        examples=[
+            ["Кој си ти?"],
+            ["Колку е 3+6/3-1?"],
+            ["Како би го населиле Марс? Биди краток."],
+            ["Напиши ми pyhon функција која прима низа броеви и проверува кои броеви се деливи со 7, оние што се деливи ги сместуваме во нова листа која ја враќаме назад."]
+        ]
+    )
+    
+if __name__ == "__main__":
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+    tokenizer.eos_token = "<|im_end|>"
+    tokenizer.pad_token = tokenizer.eos_token
+    model = AutoModelForCausalLM.from_pretrained(
+                                                 MODEL_ID,
+                                                 device_map=device,
+                                                 torch_dtype='auto',
+                                                )
+    
+    demo.queue(max_size=50).launch()