test_mistral_7b_on_cpu

Sleeping

thobuiq commited on Jan 27

Commit

2acde50

•

1 Parent(s): 2a810dd

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -16,7 +16,7 @@ model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1",
         load_in_4bit=True,
         quantization_config=bnb_config,
         torch_dtype=torch.bfloat16,
-        device_map="auto",
         trust_remote_code=True)
 # using CUDA for an optimal experience

         load_in_4bit=True,
         quantization_config=bnb_config,
         torch_dtype=torch.bfloat16,
+        device_map="cpu",
         trust_remote_code=True)
 # using CUDA for an optimal experience