PeterZentai
commited on
Commit
•
266406b
1
Parent(s):
b0ae2fb
Update README.md
Browse files
README.md
CHANGED
@@ -43,6 +43,7 @@ Cost for inference.
|
|
43 |
|
44 |
|
45 |
```python
|
|
|
46 |
from transformers import AutoTokenizer, AutoModelForCausalLM
|
47 |
|
48 |
model_id = "WeeRobots/phi-2-chat-v05"
|
@@ -50,6 +51,8 @@ model_id = "WeeRobots/phi-2-chat-v05"
|
|
50 |
model = AutoModelForCausalLM.from_pretrained(model_id, device_map={"": 0}, trust_remote_code=True)
|
51 |
tokenizer = tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True, trust_remote_code=True)
|
52 |
|
|
|
|
|
53 |
payload = tokenizer.apply_chat_template([
|
54 |
{ 'role': 'system', 'content': '''You are a state machine. The user will add state slot values and I'll keep track of them.''' },
|
55 |
{ 'role': 'user', 'content': '''Place 15 into slot apple''' },
|
@@ -58,7 +61,7 @@ payload = tokenizer.apply_chat_template([
|
|
58 |
{ 'role': 'assistant', 'content': '''Certainly''' },
|
59 |
{ 'role': 'user', 'content': '''What is value of Apples + Bananas?''' },
|
60 |
], tokenize=False, add_generation_prompt=True,)
|
61 |
-
|
62 |
model_input = tokenizer(payload, return_tensors="pt").to(device)
|
63 |
with torch.no_grad():
|
64 |
# IMPORTANT: always set the eos_token_id in this call. the model is trained to emit the eos_token the right time
|
|
|
43 |
|
44 |
|
45 |
```python
|
46 |
+
import torch
|
47 |
from transformers import AutoTokenizer, AutoModelForCausalLM
|
48 |
|
49 |
model_id = "WeeRobots/phi-2-chat-v05"
|
|
|
51 |
model = AutoModelForCausalLM.from_pretrained(model_id, device_map={"": 0}, trust_remote_code=True)
|
52 |
tokenizer = tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True, trust_remote_code=True)
|
53 |
|
54 |
+
|
55 |
+
|
56 |
payload = tokenizer.apply_chat_template([
|
57 |
{ 'role': 'system', 'content': '''You are a state machine. The user will add state slot values and I'll keep track of them.''' },
|
58 |
{ 'role': 'user', 'content': '''Place 15 into slot apple''' },
|
|
|
61 |
{ 'role': 'assistant', 'content': '''Certainly''' },
|
62 |
{ 'role': 'user', 'content': '''What is value of Apples + Bananas?''' },
|
63 |
], tokenize=False, add_generation_prompt=True,)
|
64 |
+
device = "cuda"
|
65 |
model_input = tokenizer(payload, return_tensors="pt").to(device)
|
66 |
with torch.no_grad():
|
67 |
# IMPORTANT: always set the eos_token_id in this call. the model is trained to emit the eos_token the right time
|