Update app.py

app.py

@@ -1,18 +1,22 @@
 import gradio as gr
 from gpt4all import GPT4All
 from huggingface_hub import hf_hub_download
+import subprocess
+import asyncio
 
-title = "Apollo-6B-GGUF Run On CPU"
+title = "Apollo-7B-GGUF Run On CPU"
  
 description = """
-🔎 [Apollo-6B](https://huggingface.co/FreedomIntelligence/Apollo-6B) [GGUF format model](https://huggingface.co/FreedomIntelligence/Apollo-6B-GGUF) , 8-bit quantization balanced quality gguf version, running on CPU. Using [GitHub - llama.cpp](https://github.com/ggerganov/llama.cpp) [GitHub - gpt4all](https://github.com/nomic-ai/gpt4all). 
+🔎 [Apollo-7B](https://huggingface.co/FreedomIntelligence/Apollo-7B) [GGUF format model](https://huggingface.co/FreedomIntelligence/Apollo-7B-GGUF) , 8-bit quantization balanced quality gguf version, running on CPU. Using [GitHub - llama.cpp](https://github.com/ggerganov/llama.cpp) [GitHub - gpt4all](https://github.com/nomic-ai/gpt4all). 
 
 🔨 Running on CPU-Basic free hardware. Suggest duplicating this space to run without a queue. 
 
+Mistral does not support system prompt symbol (such as ```<<SYS>>```) now, input your system prompt in the first message if you need. Learn more: [Guardrailing Mistral 7B](https://docs.mistral.ai/usage/guardrailing). 
 """
 
 """
-[Model From FreedomIntelligence/Apollo-6B-GGUF](https://huggingface.co/FreedomIntelligence/Apollo-6B-GGUF)
+[Model From TheBloke/Mistral-6B-Instruct-v0.1-GGUF](https://huggingface.co/FreedomIntelligence/Apollo-6B-GGUF)
+[Mistral-instruct-v0.1 System prompt](https://docs.mistral.ai/usage/guardrailing)
 """
 
 model_path = "models"
@@ -23,22 +27,58 @@ print("Start the model init process")
 model = model = GPT4All(model_name, model_path, allow_download = False, device="cpu")
 print("Finish the model init process")
 
-model.config["promptTemplate"] = "[INST] {0} [/INST]"
-model.config["systemPrompt"] = ""
+model.config["promptTemplate"] = "{0}"
+model.config["systemPrompt"] = "You are a multiligual AI doctor, your name is Apollo."
 model._is_chat_session_activated = False
 
 max_new_tokens = 2048
 
-def generater(message, history, temperature, top_p, top_k):
-    prompt = "<s>"
+# def generater(message, history, temperature, top_p, top_k):
+#     prompt = "<s>"
+#     for user_message, assistant_message in history:
+#         prompt += model.config["promptTemplate"].format(user_message)
+#         prompt += assistant_message + "</s>"
+#     prompt += model.config["promptTemplate"].format(message)
+#     outputs = []    
+#     for token in model.generate(prompt=prompt, temp=temperature, top_k = top_k, top_p = top_p, max_tokens = max_new_tokens, streaming=True):
+#         outputs.append(token)
+#         yield "".join(outputs)
+async def generater(message, history, temperature, top_p, top_k):
+    # 构建prompt
+    prompt = ""
     for user_message, assistant_message in history:
         prompt += model.config["promptTemplate"].format(user_message)
-        prompt += assistant_message + "</s>"
+        prompt += assistant_message
     prompt += model.config["promptTemplate"].format(message)
-    outputs = []    
-    for token in model.generate(prompt=prompt, temp=temperature, top_k = top_k, top_p = top_p, max_tokens = max_new_tokens, streaming=True):
-        outputs.append(token)
-        yield "".join(outputs)
+
+    # Debug: 打印最终的prompt以验证其正确性
+    print(f"Final prompt: {prompt}")
+
+    cmd = [
+        "./main",
+        "-m", model_path+"/"+model_name,
+        "--prompt", prompt
+    ]
+
+    # 使用subprocess.Popen调用./main并流式读取输出
+    process = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+    )
+
+    # 初始占位符输出
+    yield "Generating response..."
+    # 异步等待并处理输出
+    try:
+        while True:
+            line = process.stdout.readline()
+            if not line:
+                break  # 如果没有更多的输出，结束循环
+            print(f"Generated line: {line.strip()}")  # Debug: 打印生成的每行
+            yield line
+    except Exception as e:
+        print(f"Error during generation: {e}")
+        yield "Sorry, an error occurred while generating the response."
+
 
 def vote(data: gr.LikeData):
     if data.liked:
@@ -51,7 +91,7 @@ chatbot = gr.Chatbot(avatar_images=('resourse/user-icon.png', 'resourse/chatbot-
 additional_inputs=[
     gr.Slider(
         label="temperature",
-        value=0.8,
+        value=0.5,
         minimum=0.0,
         maximum=2.0,
         step=0.05,
@@ -95,4 +135,4 @@ with gr.Blocks(css="resourse/style/custom.css") as demo:
     iface.render()
 
 if __name__ == "__main__":
-    demo.queue(max_size=256).launch()
+    demo.queue(max_size=3).launch()