Test Successful deployment with 13b

README.md

@@ -9,7 +9,7 @@ app_file: app.py
 pinned: false
 models:
 - sentence-transformers/all-MiniLM-L6-v2
-- daryl149/llama-2-7b-chat-hf
+- daryl149/llama-2-13b-chat-hf
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -22,19 +22,19 @@ from langchain.vectorstores import FAISS
 
 
 def get_model():
-    model_repo = 'daryl149/llama-2-7b-chat-hf'
+    model_repo = 'daryl149/llama-2-13b-chat-hf'
 
     tokenizer = AutoTokenizer.from_pretrained(model_repo, use_fast=True)
 
     model = AutoModelForCausalLM.from_pretrained(
         model_repo,
         device_map='auto',
-        offload_folder="offload",
+        load_in_4bit=True,
         torch_dtype=torch.float16,
         low_cpu_mem_usage=True,
         trust_remote_code=True
     )  
-    max_len = 2048
+    max_len = 8192
 
     return tokenizer,model,max_len
 
@@ -159,4 +159,4 @@ demo = gr.ChatInterface(
 )
 
 demo.queue()
-demo.launch(debug=True)
+demo.launch(debug=True,share=True)