Update app.py

app.py

@@ -1,14 +1,19 @@
 import argparse
-import json
-
 import gradio as gr
-import requests
+
+from openai import OpenAI
+
+CLIENT = OpenAI(api_key="dummy", base_url="http://34.125.92.207:8081/v1")
 
 
 def http_bot(prompt, history):
-    headers = {"User-Agent": "vLLM Client"}
-    pload = {
-        "prompt": f"""You are SEMIKONG, an AI Assistant developed by the AI Alliance.
+    # Convert history to the format expected by the OpenAI API
+    # messages = [{"role": "user" if item['is_user'] else "assistant", "content": item['content']} for item in history]
+    messages = []
+
+    # Add the current prompt to the messages
+
+    prompt_final = f"""You are SEMIKONG, an AI Assistant developed by the AI Alliance.
 Your role is to use your's in-depth knowledge in the field of Semiconductor Manufacturing Process to answer the instruction from the user in the most sophisticate, detail and technical ways.
 You must use as much as technical term as possible to make the answer more professional and informative.
 Your answer must align with these criteria:
@@ -20,33 +25,36 @@ Your answer must align with these criteria:
 6. Comprehensive Coverage: Verify that the answer addresses all relevant aspects of the question. The response should be thorough, covering multiple angles and potential outcomes suitable for the type of question asked.
 7. Relevance to the Question: Confirm that the content of the answer directly addresses the core of the question. All explanations and solutions should be relevant and specifically tailored to the details and constraints of the question.
 Now given an instruction from : {prompt}.
-Your answer: <your answer here>
-        """,
-        "stream": True,
-        "max_tokens": 1024,
-        "temperature": 0.0,
-        "presence_penalty": 0.7,
-        "frequency_penalty": 0.5,
-        "repetition_penalty": 1.0,
-        "length_penalty": 1.0,
-        "top_p": 1.0,
-        "top_k": 50,
-        "min_p": 0.8,
-        "n": 1,
-        "best_of": 1,
-        "use_beam_search": False,
-        "early_stopping": False,
-    }
-    response = requests.post(args.model_url, headers=headers, json=pload, stream=True)
+Your answer: <your answer here>"""
+    messages.append({"role": "user", "content": prompt_final})
 
-    for chunk in response.iter_lines(
-        chunk_size=8192, decode_unicode=False, delimiter=b"\0"
-    ):
-        if chunk:
-            data = json.loads(chunk.decode("utf-8"))
-            output = data["text"][0]
-            output_clean = output.split("<your answer here>")[-1]
-            yield output_clean
+    # Call the OpenAI API to generate a response
+    response = CLIENT.chat.completions.create(
+        model="sitloboi2012/SEMIKONG-70B-v2",
+        messages=messages,
+        # stream=True,
+        max_tokens=1024,
+        temperature=0.0,
+        n=1,
+        presence_penalty=0.7,
+        top_p=1.0,
+        extra_body={
+            "frequency_penalty": 0.5,
+            "repetition_penalty": 1.0,
+            "length_penalty": 1.0,
+            "top_k": 50,
+            "min_p": 0.8,
+            "best_of": 1,
+            "use_beam_search": False,
+            "early_stopping": False,
+        },
+    )
+
+    # Extract the answer from the response
+    answer = response.choices[0].message.content
+
+    # Return the answer
+    return answer
 
 
 def build_demo():
@@ -64,10 +72,9 @@ For open-source repository, please refer to: __TBA__
 """,
         theme="soft",
         examples=[
-            "Describe different type of etching in semiconductor manufacturing process",
-            "What is Photolithography ?",
-            "Describe in detail about PVD, CVD, and ALD and their role in semiconductor manufacturing",
-            "What elements are necessary for miniaturization of semiconductor etch manufacturing equipment and what are their priorities?",
+           "Describe different type of etching in semiconductor manufacturing process",
+           "What is Photolithography ?",
+           "What elements are necessary for miniaturization of semiconductor etch manufacturing equipment and what are their priorities?",
         ],
         cache_examples=True,
         retry_btn=None,
@@ -79,12 +86,9 @@ For open-source repository, please refer to: __TBA__
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--model-url",
-        type=str,
-        default="https://u8lkrlhamyqws0-8000.proxy.runpod.net/generate",
-    )
+    parser.add_argument("--host", type=str, default=None)
+    parser.add_argument("--port", type=int, default=8001)
     args = parser.parse_args()
 
     demo = build_demo()
-    demo.queue().launch()
+    demo.queue().launch(server_name=args.host, server_port=args.port, share=True)