Update app.py

app.py

@@ -28,11 +28,11 @@ def initialize_llm(hf_token):
     return HuggingFaceLLM(
         model_name = model_name, #meta-llama/Meta-Llama-3-8B-Instruct meta-llama/Llama-2-7b-chat-hf #google/gemma-7b-it #HuggingFaceH4/zephyr-7b-beta #'GeneZC/MiniChat-2-3B'
         tokenizer_name = model_name,
-        context_window=3900,
+        context_window=1900,
         # model_kwargs={"token": hf_token, "quantization_config": quantization_config},
         model_kwargs={"token": hf_token},
         tokenizer_kwargs={"token": hf_token},
-        max_new_tokens=400,
+        max_new_tokens=300,
         device_map="auto",
     )
 
@@ -72,7 +72,7 @@ def handle_query(user_prompt, llm):
         verbose=False,
     )
     
-    MEMORY_THRESHOLD = 2500
+    MEMORY_THRESHOLD = 1900
     
     if 'memory' not in st.session_state:
         st.session_state.memory = ChatMemoryBuffer.from_defaults(token_limit=MEMORY_THRESHOLD)
@@ -125,7 +125,7 @@ def main():
 
     if 'index' not in st.session_state:
         # Load or create index
-        index, all_nodes = load_or_create_index(embed_model, directories, persist_dir)
+        index, all_nodes = load_or_create_index(st.session_state.embed_model, directories, persist_dir)
         st.session_state.index = index
         st.session_state.all_nodes_dict = {n.node_id: n for n in all_nodes}
 
@@ -147,7 +147,7 @@ def main():
     user_prompt = st.chat_input("Ask me anything:")
     if user_prompt:
         st.session_state.messages.append({'role': 'user', "content": user_prompt})
-        response = handle_query(user_prompt, llm)
+        response = handle_query(user_prompt, st.session_state.llm)
         response = response.response
         st.session_state.messages.append({'role': 'assistant', "content": response})