Add application file3

.gitignore

@@ -0,0 +1 @@
+*.ipynb

app.py

@@ -0,0 +1,156 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_name_or_path = "TheBloke/Llama-2-7b-Chat-GPTQ"
+model = AutoModelForCausalLM.from_pretrained(model_name_or_path,
+                                             device_map="auto",
+                                             trust_remote_code=False,
+                                             revision="main")
+
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
+
+
+def predict(message, chatbot, temperature=0.9, max_new_tokens=256, top_p=0.6, repetition_penalty=1.0,):
+
+    system_message = "\n당신은 도움이 되고 정중하며 정직한 Assistant입니다. 안전을 유지하면서 항상 가능한 한 도움이 되도록 답변하십시오. 귀하의 답변에는 유해하거나, 비윤리적이거나, 인종차별적이거나, 성차별적이거나, 독성이 있거나, 위험하거나 불법적인 콘텐츠가 포함되어서는 안 됩니다. 귀하의 답변은 사회적으로 편견이 없고 긍정적입니다.\n\n질문이 의미가 없거나 사실적으로 일관성이 없는 경우, 옳지 않은 것에 답변하는 대신 이유를 설명하십시오. 질문에 대한 답변을 모르는 경우, 허위정보 공유하지 마세요"    
+    input_system = f"[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n "
+
+    input_history = ""
+    for interaction in chatbot:
+        input_history = input_system + str(interaction[0]) + " [/INST] " + str(interaction[1]) + " </s><s> [INST] "
+
+    input_prompt = input_history + str(message) + " [/INST] "
+
+    inputs = tokenizer.encode(input_prompt, return_tensors="pt").to('cuda')
+
+    temperature = float(temperature)
+    if temperature < 1e-2: temperature = 1e-2
+    top_p = float(top_p)
+
+    generate_kwargs = dict(
+        input_ids=inputs,
+        temperature=temperature, 
+        top_p=top_p, 
+        max_new_tokens=max_new_tokens,
+        repetition_penalty=repetition_penalty,
+    )
+
+    outputs = model.generate(**generate_kwargs)
+    generated_indcluded_full_text = tokenizer.decode(outputs[0])
+    print("generated_indcluded_full_text:", generated_indcluded_full_text)
+
+    generated_text = generated_indcluded_full_text.split('[/INST]  ')[-1]
+    if '</s>' in generated_text : 
+        generated_text = generated_text.split('</s>')[0]
+    else : pass
+
+    import json
+    tokens = generated_text.split('\n')
+    token_list = []
+    for idx, token in enumerate(tokens):
+        token_dict = {"id": idx + 1, "text": token}
+        token_list.append(token_dict)
+    response = {"data": {"token": token_list}}
+    response = json.dumps(response, indent=4)
+
+    response = json.loads(response)
+    data_dict = response.get('data', {})
+    token_list = data_dict.get('token', [])
+
+    import time
+    partial_message = "" 
+    for token_entry in token_list:
+        if token_entry:            
+            try:
+                token_id = token_entry.get('id', None)
+                token_text = token_entry.get('text', None)
+                
+                if token_text:  
+                    for char in token_text:  
+                        partial_message += char  
+                        yield partial_message
+                        time.sleep(0.01)
+                else:
+                    gr.Warning(f"The key 'text' does not exist or is None in this token entry: {token_entry}")
+
+            except KeyError as e:
+                gr.Warning(f"KeyError: {e} occurred for token entry: {token_entry}")
+                continue
+
+title = "TheBloke/Llama-2-7b-Chat-GPTQ닝 모델 chatbot"
+
+description = """
+TheBloke/Llama-2-7b-Chat-GPTQ 모델입니다.
+"""
+css = """.toast-wrap { display: none !important } """
+examples=[
+    ['Hello there! How are you doing?'],
+    ['Can you explain to me briefly what is Python programming language?'],
+    ['Explain the plot of Cinderella in a sentence.'],
+    ['How many hours does it take a man to eat a Helicopter?'],
+    ["Write a 100-word article on 'Benefits of Open-Source in AI research'"],
+    ]
+
+import gradio as gr
+def vote(data: gr.LikeData):
+    if data.liked:
+        print("You upvoted this response: " + data.value)
+    else:
+        print("You downvoted this response: " + data.value)
+
+
+additional_inputs=[
+    gr.Slider(
+        label="Temperature",
+        value=0.9,
+        minimum=0.0,
+        maximum=1.0,
+        step=0.05,
+        interactive=True,
+        info="Higher values produce more diverse outputs",
+    ),
+    gr.Slider(
+        label="Max new tokens",
+        value=256,
+        minimum=0,
+        maximum=4096,
+        step=64,
+        interactive=True,
+        info="The maximum numbers of new tokens",
+    ),
+    gr.Slider(
+        label="Top-p (nucleus sampling)",
+        value=0.6,
+        minimum=0.0,
+        maximum=1,
+        step=0.05,
+        interactive=True,
+        info="Higher values sample more low-probability tokens",
+    ),
+    gr.Slider(
+        label="Repetition penalty",
+        value=1.2,
+        minimum=1.0,
+        maximum=2.0,
+        step=0.05,
+        interactive=True,
+        info="Penalize repeated tokens",
+    )
+]
+
+chatbot_stream = gr.Chatbot(avatar_images=('user.png', 'bot2.png'), bubble_full_width = False)
+chat_interface_stream = gr.ChatInterface(predict, 
+                 title=title, 
+                 description=description, 
+                 chatbot=chatbot_stream,
+                 css=css, 
+                 examples=examples, 
+                 cache_examples=False, 
+                 additional_inputs=additional_inputs,) 
+
+with gr.Blocks() as demo:
+
+    with gr.Tab("Streaming"):
+        chatbot_stream.like(vote, None, None)
+        chat_interface_stream.render()
+
+demo.queue(concurrency_count=75, max_size=100).launch(debug=True)

requirements.txt

@@ -0,0 +1,3 @@
+torch
+transformers
+gradio