Update app.py

app.py

@@ -9,10 +9,10 @@ access_token = os.environ["GATED_ACCESS_TOKEN"]
 quantization_config = BitsAndBytesConfig(
         load_in_4bit=True,
         bnb_4bit_quant_type="nf4",
-        bnb_4bit_compute_dtype="torch.int8",
+        bnb_4bit_compute_dtype="float16",
 )
 
-model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1", quantization_config=True, device_map="auto", token=access_token)
+model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1", quantization_config=quantization_config, device_map="auto", token=access_token)
 tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
 
 # Load the tokenizer and model