Update app.py

app.py

@@ -1,6 +1,5 @@
 import os
-import bitsandbytes
-import accelerate
+import bitsandbytes as bnb
 import torch
 import gradio as gr
 from transformers import AutoTokenizer, AutoModelForCausalLM
@@ -10,7 +9,12 @@ access_token = os.environ["GATED_ACCESS_TOKEN"]
 # Load the tokenizer and model
 model_id = "mistralai/Mixtral-8x7B-Instruct-v0.1"
 tokenizer = AutoTokenizer.from_pretrained(model_id, token=access_token)
-model = AutoModelForCausalLM.from_pretrained(model_id, load_in_4bit=True, device_map="auto", token=access_token)
+model = AutoModelForCausalLM.from_pretrained(model_id, token=access_token)
+# Initialize the quantizer
+quantizer = bnb.GemmQuantizer(act_bits=8, weight_bits=8)
+
+# Quantize the model
+model = quantizer(model)
 
 # Function to generate text using the model
 def generate_text(prompt):