Setup for runpod

app.py

@@ -1,6 +1,7 @@
 import gradio as gr
 # retrievers
 from langchain.chains import RetrievalQA
+import runpod
 
 import textwrap
 import time
@@ -49,7 +50,7 @@ repetition_penalty = 1.15
 
 pipe = pipeline(
     task = "text-generation",
-    model = model,
+    model = "daryl149/llama-2-13b-chat-hf",
     tokenizer = tokenizer,
     pad_token_id = tokenizer.eos_token_id,
     max_length = max_len,
@@ -136,27 +137,12 @@ def process_llm_response(llm_response):
     ans += "\n Sand Hill Road podcast episodes based on your question : \n" + sources_used
     return ans,sources_used
 
-def llm_ans(query):
-    start = time.time()
-    llm_response = qa_chain(query)
+def text_generation(job):
+    llm_response = qa_chain(job_input = job["prompt"])
     ans,sources_used = process_llm_response(llm_response)
-    end = time.time()
+    
+    return str(ans).replace("\n", "<br/>")
 
-    time_elapsed = int(round(end - start, 0))
-    time_elapsed_str = f'\n\nTime elapsed: {time_elapsed} s'
-    return ans, sources_used ,time_elapsed_str
 
 
-def predict(message, history):
-    # output = message # debug mode
-
-    output = str(llm_ans(message)[0]).replace("\n", "<br/>")
-    return output
-
-demo = gr.ChatInterface(
-    predict,
-    title = f' Sand Hill Road Podcast Chatbot'
-)
-
-demo.queue()
-demo.launch(debug=True,share=True)
+runpod.serverless.start({"handler": text_generation})

requirements.txt

@@ -6,4 +6,5 @@ sentence_transformers
 accelerate
 bitsandbytes
 xformers
+runpod
 einops