add cuda stuff

app.py

@@ -1,6 +1,4 @@
-import gradio as gr
-from huggingface_hub import InferenceClient
-
+# main.py
 import spaces
 import torch
 import torch.nn.functional as F
@@ -12,11 +10,17 @@ import queue
 import os
 import json
 import numpy as np
-
+import gradio as gr
+from huggingface_hub import InferenceClient
 import openai
 from openai import OpenAI
 from globalvars import API_BASE, API_KEY, intention_prompt
 
+os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:30'
+os.environ['CUDA_LAUNCH_BLOCKING'] = '1'
+os.environ['CUDA_CACHE_DISABLE'] = '1'
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
 ## add chroma vector store