mobiuslabsgmbh
/

Mixtral-8x7B-Instruct-v0.1-hf-2bit_g16_s128-HQQ

Text Generation

Mixture of Experts

text-generation-inference

Model card Files Files and versions Community

mobicham commited on Dec 11, 2023

Commit

ac3c4e8

•

1 Parent(s): 26c76e9

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -19,7 +19,7 @@ model     = HQQModelForCausalLM.from_quantized(model_id)
 HQQLinear.set_backend(HQQBackend.PYTORCH_COMPILE) #Optional
 #Text Generation
-prompt = "<s> [INST] Who is Elon Musk? [/INST]"
 inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False)
 outputs = model.generate(**(inputs.to('cuda')), max_new_tokens=1000)

 HQQLinear.set_backend(HQQBackend.PYTORCH_COMPILE) #Optional
 #Text Generation
+prompt = "<s> [INST] How do I build a car? [/INST] "
 inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False)
 outputs = model.generate(**(inputs.to('cuda')), max_new_tokens=1000)