google/gemma-7b-it

app.py

@@ -8,7 +8,7 @@ import streamlit as st
 
 
 if "model" not in st.session_state:
-    st.session_state["model"] = "TheBloke/Mistral-7B-Instruct-v0.2-GGUF"
+    st.session_state["model"] = "google/gemma-7b-it"
 
 # Initialize chat history
 if "messages" not in st.session_state:

model.py

@@ -1,4 +1,11 @@
-from ctransformers import AutoModelForCausalLM 
-model = AutoModelForCausalLM.from_pretrained("TheBloke/Mistral-7B-OpenOrca-GGUF", model_file="mistral-7b-openorca.Q2_K.gguf", model_type="mistral")
+#from ctransformers import AutoModelForCausalLM 
+#model = AutoModelForCausalLM.from_pretrained("TheBloke/Mistral-7B-OpenOrca-GGUF", model_file="mistral-7b-openorca.Q2_K.gguf", model_type="mistral")
 
-#model = AutoModelForCausalLM.from_pretrained("TheBloke/Mistral-7B-Instruct-v0.2-GGUF")
+#model = AutoModelForCausalLM.from_pretrained("TheBloke/Mistral-7B-Instruct-v0.2-GGUF")
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+
+#quantization_config = BitsAndBytesConfig(load_in_4bit=True)
+
+tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b-it")
+model = AutoModelForCausalLM.from_pretrained("google/gemma-7b-it")

requirements.txt

@@ -2,5 +2,6 @@ streamlit==1.31.1
 pymupdf==1.23.18
 chromadb==0.4.22
 langchain==0.1.3
-sentence-transformers==2.2.2
-ctransformers
+transformers==4.38.2
+accelerate==0.27.2
+#ctransformers