gemma-2-9b-it

Runtime error

ehristoforu commited on Jul 22

Commit

70b5193

•

1 Parent(s): c528c03

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -33,7 +33,7 @@ tokenizer = GemmaTokenizerFast.from_pretrained(model_id)
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     device_map="auto",
-    #quantization_config=BitsAndBytesConfig(load_in_8bit=True),
 )
 model.config.sliding_window = 4096
 model.eval()

 model = AutoModelForCausalLM.from_pretrained(
     model_id,
     device_map="auto",
+    quantization_config=BitsAndBytesConfig(load_in_4bit=True),
 )
 model.config.sliding_window = 4096
 model.eval()