jeremyarancio
commited on
Commit
•
39ec5b7
1
Parent(s):
dc32044
Remove return dict from model import
Browse files- handler.py +1 -1
handler.py
CHANGED
@@ -11,7 +11,7 @@ LOGGER = logging.getLogger(__name__)
|
|
11 |
class EndpointHandler():
|
12 |
def __init__(self, path=""):
|
13 |
config = PeftConfig.from_pretrained(path)
|
14 |
-
model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path,
|
15 |
self.tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
|
16 |
# Load the Lora model
|
17 |
self.model = PeftModel.from_pretrained(model, path)
|
|
|
11 |
class EndpointHandler():
|
12 |
def __init__(self, path=""):
|
13 |
config = PeftConfig.from_pretrained(path)
|
14 |
+
model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, load_in_8bit=True, device_map='auto')
|
15 |
self.tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
|
16 |
# Load the Lora model
|
17 |
self.model = PeftModel.from_pretrained(model, path)
|