Spaces:

AnishHF
/

Mistral-7B

Runtime error

AnishHF commited on May 22

Commit

27f4932

•

1 Parent(s): f11657e

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -11,7 +11,7 @@ model_id = "mistralai/Mixtral-8x7B-Instruct-v0.1"
 tokenizer = AutoTokenizer.from_pretrained(model_id, token=access_token)
 model = AutoModelForCausalLM.from_pretrained(model_id, token=access_token)
 # Initialize the quantizer
-quantizer = bnb.GemmQuantizer(act_bits=8, weight_bits=8)
 # Quantize the model
 model = quantizer(model)

 tokenizer = AutoTokenizer.from_pretrained(model_id, token=access_token)
 model = AutoModelForCausalLM.from_pretrained(model_id, token=access_token)
 # Initialize the quantizer
+quantizer = bnb.GemmQuantizer(act_bits=4, weight_bits=4)
 # Quantize the model
 model = quantizer(model)