Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
@@ -11,7 +11,7 @@ model_id = "mistralai/Mixtral-8x7B-Instruct-v0.1"
|
|
11 |
tokenizer = AutoTokenizer.from_pretrained(model_id, token=access_token)
|
12 |
model = AutoModelForCausalLM.from_pretrained(model_id, token=access_token)
|
13 |
# Initialize the quantizer
|
14 |
-
quantizer = bnb.GemmQuantizer(act_bits=
|
15 |
|
16 |
# Quantize the model
|
17 |
model = quantizer(model)
|
|
|
11 |
tokenizer = AutoTokenizer.from_pretrained(model_id, token=access_token)
|
12 |
model = AutoModelForCausalLM.from_pretrained(model_id, token=access_token)
|
13 |
# Initialize the quantizer
|
14 |
+
quantizer = bnb.GemmQuantizer(act_bits=4, weight_bits=4)
|
15 |
|
16 |
# Quantize the model
|
17 |
model = quantizer(model)
|