Update llm_handler.py
Browse files- llm_handler.py +6 -6
llm_handler.py
CHANGED
@@ -4,7 +4,7 @@ from llama_cpp_agent import MessagesFormatterType
|
|
4 |
from llama_cpp_agent.providers import LlamaCppPythonProvider
|
5 |
|
6 |
# Initialize the Llama model
|
7 |
-
llama_model = Llama("Arcee-Spark-GGUF/Arcee-Spark-Q4_K_M.gguf", n_batch=1024, n_threads=
|
8 |
|
9 |
# Create the provider
|
10 |
provider = LlamaCppPythonProvider(llama_model)
|
@@ -12,7 +12,7 @@ provider = LlamaCppPythonProvider(llama_model)
|
|
12 |
# Create the agent
|
13 |
agent = LlamaCppAgent(
|
14 |
provider,
|
15 |
-
system_prompt="You are a helpful assistant.",
|
16 |
predefined_messages_formatter_type=MessagesFormatterType.CHATML,
|
17 |
debug_output=True
|
18 |
)
|
@@ -24,19 +24,19 @@ settings.stream = True
|
|
24 |
|
25 |
def send_to_llm(provider, msg_list):
|
26 |
try:
|
27 |
-
|
28 |
full_message = "\n".join([f"{msg['role']}: {msg['content']}" for msg in msg_list])
|
29 |
|
30 |
-
|
31 |
response = agent.get_chat_response(full_message, llm_sampling_settings=settings)
|
32 |
|
33 |
-
|
34 |
if isinstance(response, str):
|
35 |
return response, None
|
36 |
elif hasattr(response, 'content'):
|
37 |
return response.content, None
|
38 |
else:
|
39 |
-
return str(response), None
|
40 |
except Exception as e:
|
41 |
print(f"Error in send_to_llm: {str(e)}")
|
42 |
return f"Error: {str(e)}", None
|
|
|
4 |
from llama_cpp_agent.providers import LlamaCppPythonProvider
|
5 |
|
6 |
# Initialize the Llama model
|
7 |
+
llama_model = Llama("Arcee-Spark-GGUF/Arcee-Spark-Q4_K_M.gguf", n_batch=1024, n_threads=24, n_gpu_layers=33, n_ctx=2048, verbose=False)
|
8 |
|
9 |
# Create the provider
|
10 |
provider = LlamaCppPythonProvider(llama_model)
|
|
|
12 |
# Create the agent
|
13 |
agent = LlamaCppAgent(
|
14 |
provider,
|
15 |
+
system_prompt="You are a helpful assistant who's purpose is it to help users craft and edit datasets.",
|
16 |
predefined_messages_formatter_type=MessagesFormatterType.CHATML,
|
17 |
debug_output=True
|
18 |
)
|
|
|
24 |
|
25 |
def send_to_llm(provider, msg_list):
|
26 |
try:
|
27 |
+
|
28 |
full_message = "\n".join([f"{msg['role']}: {msg['content']}" for msg in msg_list])
|
29 |
|
30 |
+
|
31 |
response = agent.get_chat_response(full_message, llm_sampling_settings=settings)
|
32 |
|
33 |
+
|
34 |
if isinstance(response, str):
|
35 |
return response, None
|
36 |
elif hasattr(response, 'content'):
|
37 |
return response.content, None
|
38 |
else:
|
39 |
+
return str(response), None
|
40 |
except Exception as e:
|
41 |
print(f"Error in send_to_llm: {str(e)}")
|
42 |
return f"Error: {str(e)}", None
|