Update llm_handler.py

llm_handler.py

@@ -2,15 +2,18 @@ import requests
 import json
 from openai import OpenAI
 from params import OPENAI_MODEL, OPENAI_API_KEY
-from llama_cpp import Llama
-from llama_cpp_agent import LlamaCppAgent
-from llama_cpp_agent import MessagesFormatterType
-from llama_cpp_agent.providers import LlamaCppPythonProvider
+import llamanet
 
 # Add this at the top of the file
 local_model_base_url = "http://localhost:11434/v1"
 anything_llm_workspace = "<input-workspace-name-here>"
 
+# Initialize llamanet
+llamanet.run()
+
+# Create an instance of the OpenAI class
+client = OpenAI()
+
 def set_local_model_base_url(url):
     global local_model_base_url
     local_model_base_url = url
@@ -19,35 +22,22 @@ def set_anything_llm_workspace(workspace):
     global anything_llm_workspace
     anything_llm_workspace = workspace
 
-# Create an instance of the OpenAI class for the local model
-client = OpenAI(api_key="local-model", base_url=local_model_base_url)
-
-# Initialize LlamaCpp model and agent
-llama_model = Llama("Arcee-Spark-GGUF/Arcee-Spark-Q4_K_M.gguf", n_batch=1024, n_threads=24, n_gpu_layers=33, n_ctx=4098, verbose=False)
-provider = LlamaCppPythonProvider(llama_model)
-llama_agent = LlamaCppAgent(
-    provider,
-    system_prompt="You are a helpful assistant.",
-    predefined_messages_formatter_type=MessagesFormatterType.MISTRAL,
-    debug_output=True
-)
-
-# Configure provider settings
-settings = provider.get_provider_default_settings()
-settings.max_tokens = 2000
-settings.stream = True
-
 def send_to_chatgpt(msg_list):
     try:
-        # Update the send_to_chatgpt function to use the dynamic base_url
-        client = OpenAI(api_key="local-model", base_url=local_model_base_url)
         completion = client.chat.completions.create(
-            model=OPENAI_MODEL,
+            model='https://huggingface.co/arcee-ai/Arcee-Spark-GGUF/blob/main/Arcee-Spark-IQ4_XS.gguf',  # This will use the llamanet model
+            messages=msg_list,
             temperature=0.6,
-            messages=msg_list
+            stream=True
         )
-        chatgpt_response = completion.choices[0].message.content
-        chatgpt_usage = completion.usage
+        
+        chatgpt_response = ""
+        for chunk in completion:
+            if chunk.choices[0].delta.content is not None:
+                chatgpt_response += chunk.choices[0].delta.content
+        
+        # Note: Usage information might not be available with llamanet
+        chatgpt_usage = None
         return chatgpt_response, chatgpt_usage
     except Exception as e:
         print(f"Error in send_to_chatgpt: {str(e)}")
@@ -77,24 +67,10 @@ def send_to_anything_llm(msg_list):
         print(f"Error in send_to_anything_llm: {str(e)}")
         return f"Error: {str(e)}", None
 
-def send_to_llamacpp(msg_list):
-    try:
-        # Convert the message list to the format expected by LlamaCppAgent
-        formatted_messages = [{"role": msg["role"], "content": msg["content"]} for msg in msg_list]
-        response = llama_agent(formatted_messages, settings=settings)
-        chatgpt_response = response.message.content
-        chatgpt_usage = {"prompt_tokens": response.usage.prompt_tokens, "completion_tokens": response.usage.completion_tokens, "total_tokens": response.usage.total_tokens}
-        return chatgpt_response, chatgpt_usage
-    except Exception as e:
-        print(f"Error in send_to_llamacpp: {str(e)}")
-        return f"Error: {str(e)}", None
-
 def send_to_llm(provider, msg_list):
     if provider == "local-model":
         return send_to_chatgpt(msg_list)
     elif provider == "anything-llm":
         return send_to_anything_llm(msg_list)
-    elif provider == "llamacpp":
-        return send_to_llamacpp(msg_list)
     else:
-        raise ValueError(f"Unknown provider: {provider}")
+        raise ValueError(f"Unknown provider: {provider}")